diff --git a/exec-git.js b/exec-git.js index 0019457..1bd6edf 100644 --- a/exec-git.js +++ b/exec-git.js @@ -1,4 +1,4 @@ -var Promise = require('rsvp').Promise; +var Promise = require('bluebird').Promise; var exec = require('child_process').exec; var os = require('os'); diff --git a/github.js b/github.js index 88b5a68..1a02af7 100644 --- a/github.js +++ b/github.js @@ -5,14 +5,12 @@ var rimraf = require('rimraf'); var request = require('request'); var expandTilde = require('expand-tilde'); -var Promise = require('rsvp').Promise; -var asp = require('rsvp').denodeify; +var Promise = require('bluebird'); +var asp = require('bluebird').Promise.promisify; -var tar = require('tar'); +var tar = require('tar-fs'); var zlib = require('zlib'); -var yauzl = require('yauzl'); - var semver = require('semver'); var which = require('which'); @@ -48,12 +46,13 @@ function createRemoteStrings(auth, hostname) { // avoid storing passwords as plain text in config function encodeCredentials(auth) { - return new Buffer(encodeURIComponent(auth.username) + ':' + encodeURIComponent(auth.password)).toString('base64'); + return new Buffer(auth.username + ':' + auth.password).toString('base64'); } function decodeCredentials(str) { - var auth = new Buffer(str, 'base64').toString('ascii').split(':'); + var auth = new Buffer(str, 'base64').toString('utf8').split(':'); var username, password; + try { username = decodeURIComponent(auth[0]); password = decodeURIComponent(auth[1]); @@ -118,7 +117,7 @@ var GithubLocation = function(options, ui) { this.execOpt = { cwd: options.tmpDir, - timeout: options.timeout * 1000, + timeout: options.timeouts.download * 1000, killSignal: 'SIGKILL', maxBuffer: this.max_repo_size || 2 * 1024 * 1024, env: extend({}, process.env) @@ -263,31 +262,7 @@ function configureCredentials(config, ui) { }); } -function checkRateLimit(headers) { - if (headers.status.match(/^401/)) - throw 'Unauthorized response for GitHub API.\n' + - 'Use %jspm registry config github% to reconfigure the credentials, or update them in your ~/.netrc file.'; - if (headers.status.match(/^406/)) - throw 'Unauthorized response for GitHub API.\n' + - 'If using an access token ensure it has public_repo access.\n' + - 'Use %jspm registry config github% to configure the credentials, or add them to your ~/.netrc file.'; - - if (headers['x-ratelimit-remaining'] != '0') - return; - - var remaining = (headers['x-ratelimit-reset'] * 1000 - new Date(headers.date).getTime()) / 60000; - - if (this.auth) - return Promise.reject('\nGitHub rate limit reached, with authentication enabled.' + - '\nThe rate limit will reset in `' + Math.round(remaining) + ' minutes`.'); - - var err = new Error('GitHub rate limit reached.'); - err.config = true; - err.hideStack = true; - - return Promise.reject(err); -} - +var apiWarned = false; // static configuration function GithubLocation.configure = function(config, ui) { @@ -319,8 +294,7 @@ GithubLocation.configure = function(config, ui) { }); }; -// regular expression to verify package names -GithubLocation.packageFormat = /^[^\/]+\/[^\/]+/; +GithubLocation.packageNameFormats = ['*/*']; GithubLocation.prototype = { @@ -432,7 +406,10 @@ GithubLocation.prototype = { if (meta.vPrefix) version = 'v' + version; - return asp(request)(extend({ + var self = this; + var ui = this.ui; + + return asp(request)({ uri: this.apiRemoteString + 'repos/' + repo + '/contents/package.json', headers: { 'User-Agent': 'jspm', @@ -440,23 +417,37 @@ GithubLocation.prototype = { }, qs: { ref: version + }, + strictSSL: this.defaultRequestOptions.strictSSL + }).then(function(res) { + // API auth failure warnings + function apiFailWarn(reason, showAuthCommand) { + if (apiWarned) + return; + + ui.log('warn', 'Unable to use the GitHub API to speed up dependency downloads due to ' + reason + + (showAuthCommand ? '\nTo resolve use %jspm registry config github% to configure the credentials, or update them in your ~/.netrc file.' : '')); + apiWarned = true; } - }, this.defaultRequestOptions - )).then(function(res) { - var rateLimitResponse = checkRateLimit.call(this, res.headers); - if (rateLimitResponse) - return rateLimitResponse; - - if (res.statusCode == 404) { - // it is quite valid for a repo not to have a package.json - return {}; + + if (res.statusCode === 401) + return apiFailWarn('lack of authorization', true); + if (res.statusCode === 406) + return apiFailWarn('insufficient permissions. Ensure you have public_repo access.'); + if (res.headers['x-ratelimit-remaining'] == '0') { + if (self.auth) + return apiFailWarn('the rate limit being reached, which will be reset in `' + + Math.round((res.headers['x-ratelimit-reset'] * 1000 - new Date(res.headers.date).getTime()) / 60000) + ' minutes`.'); + return apiFailWarn('the rate limit being reached.', true); } - if (res.statusCode != 200) - throw 'Unable to check repo package.json for release, status code ' + res.statusCode; + return apiFailWarn('invalid response code ' + res.statusCode + '.'); - var packageJSON; + // it is quite valid for a repo not to have a package.json + if (res.statusCode == 404) + return {}; + var packageJSON; try { packageJSON = JSON.parse(res.body); } @@ -468,19 +459,24 @@ GithubLocation.prototype = { }); }, - processPackageConfig: function(pjson, packageName) { - if (!pjson.jspm || !pjson.jspm.files) - delete pjson.files; + processPackageConfig: function(packageConfig, packageName) { + if (!packageConfig.jspm || !packageConfig.jspm.files) + delete packageConfig.files; var self = this; - if (pjson.dependencies && !pjson.registry && (!pjson.jspm || !pjson.jspm.dependencies)) { + if ((packageConfig.dependencies || packageConfig.peerDependencies || packageConfig.optionalDependencies) && + !packageConfig.registry && (!packageConfig.jspm || !(packageConfig.jspm.dependencies || packageConfig.jspm.peerDependencies || packageConfig.jspm.optionalDependencies))) { var hasDependencies = false; - for (var p in pjson.dependencies) + for (var p in packageConfig.dependencies) + hasDependencies = true; + for (var p in packageConfig.peerDependencies) + hasDependencies = true; + for (var p in packageConfig.optionalDependencies) hasDependencies = true; if (packageName && hasDependencies) { - var looksLikeNpm = pjson.name && pjson.version && (pjson.description || pjson.repository || pjson.author || pjson.license || pjson.scripts); + var looksLikeNpm = packageConfig.name && packageConfig.version && (packageConfig.description || packageConfig.repository || packageConfig.author || packageConfig.license || packageConfig.scripts); var isSemver = semver.valid(packageName.split('@').pop()); var noDepsMsg; @@ -489,35 +485,45 @@ GithubLocation.prototype = { if (!isSemver) noDepsMsg = 'To install this package as it would work on npm, install with a registry override via %jspm install ' + packageName + ' -o "{registry:\'npm\'}"%.' else - noDepsMsg = 'If the dependencies aren\'t needed ignore this message. Alternatively set a `registry` or `dependencies` override or use the npm registry version at %jspm install npm:' + pjson.name + '@^' + pjson.version + '% instead.'; + noDepsMsg = 'If the dependencies aren\'t needed ignore this message. Alternatively set a `registry` or `dependencies` override or use the npm registry version at %jspm install npm:' + packageConfig.name + '@^' + packageConfig.version + '% instead.'; } else { noDepsMsg = 'If this is your own package, add `"registry": "jspm"` to the package.json to ensure the dependencies are installed.' } if (noDepsMsg) { - delete pjson.dependencies; + delete packageConfig.dependencies; + delete packageConfig.peerDependencies; + delete packageConfig.optionalDependencies; this.ui.log('warn', '`' + packageName + '` dependency installs skipped as it\'s a GitHub package with no registry property set.\n' + noDepsMsg + '\n'); } } else { - delete pjson.dependencies; + delete packageConfig.dependencies; + delete packageConfig.peerDependencies; + delete packageConfig.optionalDependencies; } } // on GitHub, single package names ('jquery') are from jspm registry // double package names ('components/jquery') are from github registry - if (!pjson.registry) { - for (var d in pjson.dependencies) { - var depName = pjson.dependencies[d]; + if (!packageConfig.registry || packageConfig.registry == 'github') { + for (var d in packageConfig.dependencies) + packageConfig.dependencies[d] = convertDependency(d, packageConfig.dependencies[d]); + for (var d in packageConfig.peerDependencies) + packageConfig.peerDependencies[d] = convertDependency(d, packageConfig.peerDependencies[d]); + for (var d in packageConfig.optionalDependencies) + packageConfig.optionalDependencies[d] = convertDependency(d, packageConfig.optionalDependencies[d]); + + function convertDependency(d, depName) { var depVersion; if (depName.indexOf(':') != -1) - continue; + return depName; if (depName.indexOf('@') != -1) { - depName = depName.substr(0, depName.indexOf('@')); depVersion = depName.substr(depName.indexOf('@') + 1); + depName = depName.substr(0, depName.indexOf('@')); } else { depVersion = depName; @@ -525,10 +531,12 @@ GithubLocation.prototype = { } if (depName.split('/').length == 1) - pjson.dependencies[d] = 'jspm:' + depName + (depVersion && depVersion !== true ? '@' + depVersion : ''); + return 'jspm:' + depName + (depVersion && depVersion !== true ? '@' + depVersion : ''); + + return depName + '@' + depVersion; } } - return pjson; + return packageConfig; }, download: function(repo, version, hash, meta, outDir) { @@ -541,256 +549,55 @@ GithubLocation.prototype = { var self = this; - return this.checkReleases(repo, version) - .then(function(release) { - if (!release) - return true; - - // Download from the release archive - return new Promise(function(resolve, reject) { - var inPipe; - - if (release.type == 'tar') { - (inPipe = zlib.createGunzip()) - .pipe(tar.Extract({ - path: outDir, - strip: 0, - filter: function() { - return !this.type.match(/^.*Link$/); - } - })) - .on('end', function() { - resolve(); - }) - .on('error', reject); - } - else if (release.type == 'zip') { - var tmpDir = path.resolve(execOpt.cwd, 'release-' + repo.replace('/', '#') + '-' + version); - var tmpFile = tmpDir + '.' + release.type; - - var repoDir; - - inPipe = fs.createWriteStream(tmpFile) - .on('finish', function() { - return clearDir(tmpDir) - .then(function() { - return asp(fs.mkdir(tmpDir)); - }) - .then(function() { - return new Promise(function(resolve, reject) { - var files = []; - yauzl.open(tmpFile, function(err, zipFile) { - if (err) - return reject(err); - - zipFile.on('entry', function(entry) { - var fileName = tmpDir + '/' + entry.fileName; - - if (fileName[fileName.length - 1] == '/') - return; - - zipFile.openReadStream(entry, function(err, readStream) { - if (err) - return reject(err); - mkdirp(path.dirname(fileName), function(err) { - if (err) - return reject(err); - files.push(new Promise(function(_resolve, _reject) { - var p = fs.createWriteStream(fileName).on("close", function(err) { - if (err) _reject(err); - _resolve(); - }); - readStream.pipe(p); - })); - }); - }); - }); - zipFile.on('close', function() { - Promise.all(files).then(function() { - resolve(); - }).catch(function(e) { - reject(e); - }); - }); - }); - - - }) - }) - .then(function() { - return checkStripDir(tmpDir); - }) - .then(function(_repoDir) { - repoDir = _repoDir; - return asp(fs.rmdir)(outDir); - }) - .then(function() { - return asp(fs.rename)(repoDir, outDir); - }) - .then(function() { - return asp(fs.unlink)(tmpFile); - }) - .then(resolve, reject); - }) - .on('error', reject); - } - else { - throw 'GitHub release found, but no archive present.'; - } - - // now that the inPipe is ready, do the request - request(extend({ - uri: release.url, - headers: { - 'accept': 'application/octet-stream', - 'user-agent': 'jspm' - }, - followRedirect: false, - auth: self.auth && { - user: self.auth.username, - pass: self.auth.password - } - }, self.defaultRequestOptions - )).on('response', function(archiveRes) { - var rateLimitResponse = checkRateLimit.call(this, archiveRes.headers); - if (rateLimitResponse) - return rateLimitResponse.then(resolve, reject); - - if (archiveRes.statusCode != 302) - return reject('Bad response code ' + archiveRes.statusCode + '\n' + JSON.stringify(archiveRes.headers)); - - request(extend({ - uri: archiveRes.headers.location, headers: { - 'accept': 'application/octet-stream', - 'user-agent': 'jspm' - } - }, self.defaultRequestOptions - )) - .on('response', function(archiveRes) { - - if (max_repo_size && archiveRes.headers['content-length'] > max_repo_size) - return reject('Response too large.'); - - archiveRes.pause(); + // Download from the git archive + return new Promise(function(resolve, reject) { + request({ + uri: remoteString + repo + '/archive/' + version + '.tar.gz', + headers: { 'accept': 'application/octet-stream' }, + strictSSL: self.defaultRequestOptions.strictSSL + }) + .on('response', function(pkgRes) { + if (pkgRes.statusCode != 200) + return reject('Bad response code ' + pkgRes.statusCode); - archiveRes.pipe(inPipe); + if (max_repo_size && pkgRes.headers['content-length'] > max_repo_size) + return reject('Response too large.'); - archiveRes.on('error', reject); + pkgRes.pause(); - archiveRes.resume(); + var gzip = zlib.createGunzip(); - }) - .on('error', reject); - }) + pkgRes + .pipe(gzip) + .pipe(tar.extract(outDir, { + strip: 1, + filter: function(_, header) { + return header.type !== 'file' && header.type !== 'directory' + } + }).on('finish', resolve).on('error', reject)) .on('error', reject); - }); - }) - .then(function(git) { - if (!git) - return; - // Download from the git archive - return new Promise(function(resolve, reject) { - request(extend({ - uri: remoteString + repo + '/archive/' + version + '.tar.gz', - headers: { 'accept': 'application/octet-stream' } - }, self.defaultRequestOptions - )) - .on('response', function(pkgRes) { - if (pkgRes.statusCode != 200) - return reject('Bad response code ' + pkgRes.statusCode); - - if (max_repo_size && pkgRes.headers['content-length'] > max_repo_size) - return reject('Response too large.'); - - pkgRes.pause(); - - var gzip = zlib.createGunzip(); - - pkgRes - .pipe(gzip) - .pipe(tar.Extract({ - path: outDir, - strip: 1, - filter: function() { - return !this.type.match(/^.*Link$/); - } - })) - .on('error', reject) - .on('end', resolve); - - pkgRes.resume(); - - }) - .on('error', reject); - }); - }); - }, + pkgRes.resume(); - checkReleases: function(repo, version) { - // NB cache this on disk with etags - var reqOptions = extend({ - uri: this.apiRemoteString + 'repos/' + repo + '/releases', - headers: { - 'User-Agent': 'jspm', - 'Accept': 'application/vnd.github.v3+json' - }, - followRedirect: false - }, this.defaultRequestOptions); - - return asp(request)(reqOptions) - .then(function(res) { - var rateLimitResponse = checkRateLimit.call(this, res.headers); - if (rateLimitResponse) - return rateLimitResponse; - return Promise.resolve() - .then(function() { - try { - return JSON.parse(res.body); - } - catch(e) { - throw 'Unable to parse GitHub API response'; - } }) - .then(function(releases) { - // run through releases list to see if we have this version tag - for (var i = 0; i < releases.length; i++) { - var tagName = (releases[i].tag_name || '').trim(); - - if (tagName == version) { - var firstAsset = releases[i].assets.filter(function(asset) { - if (asset.name.substr(asset.name.length - 7, 7) == '.tar.gz' || asset.name.substr(asset.name.length - 4, 4) == '.tgz') - asset.fileType = 'tar'; - else if (asset.name.substr(asset.name.length - 4, 4) == '.zip') - asset.fileType = 'zip'; - return !!asset.fileType; - }) - .sort(function(asset) { - // src.zip comes after file.zip - return asset.name.indexOf('src') == -1 ? -1 : 1; - })[0]; - - if (!firstAsset) - return false; - - return { url: firstAsset.url, type: firstAsset.fileType }; - } - } - return false; + .on('error', function(err) { + if (err.code == 'ECONNRESET') + err.retriable = true; + throw err; }); }); }, // check if the main entry point exists. If not, try the bower.json main. - build: function(pjson, dir) { - var main = pjson.main || ''; - var libDir = pjson.directories && (pjson.directories.dist || pjson.directories.lib) || '.'; + processPackage: function(packageConfig, packageName, dir) { + var main = packageConfig.main || dir.split('/').pop().split('@').slice(0, -1).join('@') + (dir.substr(dir.length - 3, 3) != '.js' ? '.js' : ''); + var libDir = packageConfig.directories && (packageConfig.directories.dist || packageConfig.directories.lib) || '.'; if (main instanceof Array) main = main[0]; if (typeof main != 'string') - return; + return packageConfig; // convert to windows-style paths if necessary main = main.replace(/\//g, path.sep); @@ -816,7 +623,7 @@ GithubLocation.prototype = { return checkMain(main, libDir) .then(function(hasMain) { if (hasMain) - return; + return hasMain; return asp(fs.readFile)(path.resolve(dir, 'bower.json')) .then(function(bowerJson) { @@ -834,11 +641,17 @@ GithubLocation.prototype = { return checkMain(main); }, function() {}) .then(function(hasBowerMain) { - if (!hasBowerMain) - return; + if (hasBowerMain) + return hasBowerMain; - pjson.main = main; + main = 'index'; + return checkMain(main, libDir); }); + }) + .then(function(hasMain) { + if (hasMain) + packageConfig.main = main.replace(/\\/g, '/'); + return packageConfig; }); } diff --git a/package.json b/package.json index f9c7c2c..54c021a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jspm-github", - "version": "0.13.13", + "version": "0.14.14", "description": "jspm GitHub endpoint", "main": "github.js", "scripts": { @@ -16,17 +16,16 @@ "url": "https://github.com/jspm/github/issues" }, "dependencies": { + "bluebird": "^3.5.0", "expand-tilde": "^1.2.0", "graceful-fs": "^4.1.3", - "mkdirp": "~0.5.0", + "mkdirp": "^0.5.1", "netrc": "^0.1.3", - "request": "~2.53.0", - "rimraf": "~2.3.2", - "rsvp": "^3.0.17", + "request": "^2.74.0", + "rimraf": "^2.6.1", "semver": "^5.0.1", - "tar": "^2.2.1", - "which": "^1.0.9", - "yauzl": "^2.3.1" + "tar-fs": "^1.15.3", + "which": "^1.0.9" }, "homepage": "https://github.com/jspm/github", "devDependencies": {