From 325409c69fa928380f0ad9b232551a5806dfb58f Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 25 Oct 2018 19:54:17 +0200 Subject: [PATCH] Add more hooks (#640) Fixes #625 Fixes #634 --- readme.md | 83 ++++++++- source/as-promise.js | 17 ++ source/index.js | 5 +- source/known-hook-events.js | 7 +- source/normalize-arguments.js | 36 ++-- source/request-as-event-emitter.js | 33 +++- test/hooks.js | 288 ++++++++++++++++++++++------- 7 files changed, 377 insertions(+), 92 deletions(-) diff --git a/readme.md b/readme.md index b25163b41..24310fb01 100644 --- a/readme.md +++ b/readme.md @@ -317,8 +317,7 @@ got('sindresorhus.com', { ###### hooks -Type: `Object`
-Default: `{beforeRequest: []}` +Type: `Object` Hooks allow modifications during the request lifecycle. Hook functions may be async and are run serially. @@ -327,11 +326,87 @@ Hooks allow modifications during the request lifecycle. Hook functions may be as Type: `Function[]`
Default: `[]` -Called with the normalized request options. Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) and [`got.create()`](advanced-creation.md) when you want to create an API client that, for example, uses HMAC-signing. +Called with [normalized](source/normalize-arguments.js) [request options](#options). Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) and [`got.create()`](advanced-creation.md) when you want to create an API client that, for example, uses HMAC-signing. See the [AWS section](#aws) for an example. -**Note**: Modifying the `body` is not recommended because the `content-length` header has already been computed and assigned. +**Note**: If you modify the `body` you will need to modify the `content-length` header too, because it has already been computed and assigned. + +###### hooks.beforeRedirect + +Type: `Function[]`
+Default: `[]` + +Called with [normalized](source/normalize-arguments.js) [request options](#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. Example: + +```js +const got = require('got'); + +got('example.com', { + hooks: { + beforeRedirect: [ + options => { + if (options.hostname === 'deadSite') { + options.hostname = 'fallbackSite'; + } + } + ] + } +}); +``` + +###### hooks.beforeRetry + +Type: `Function[]`
+Default: `[]` + +Called with [normalized](source/normalize-arguments.js) [request options](#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. Example: + +```js +const got = require('got'); + +got('example.com', { + hooks: { + beforeRetry: [ + (options, error, retryCount) => { + if (error.statusCode === 413) { // Payload too large + options.body = getNewBody(); + } + } + ] + } +}); +``` + +###### hooks.afterResponse + +Type: `Function[]`
+Default: `[]` + +Called with [response object](#response). Each function should return the response or updated options. This is especially useful when you want to refresh an access token. Example: + +```js +const got = require('got'); + +got('example.com', { + hooks: { + afterResponse: [ + response => { + if (response.statusCode === 401) { // Unauthorized + return { + headers: { + token: getNewToken(); // Refresh the access token + } + }; + } + + // No changes otherwise + return response; + } + ] + } +}); +``` #### Response diff --git a/source/as-promise.js b/source/as-promise.js index 2110a3f61..931f2842d 100644 --- a/source/as-promise.js +++ b/source/as-promise.js @@ -32,6 +32,23 @@ module.exports = options => { response.body = data; + try { + for (const hook of options.hooks.afterResponse) { + // eslint-disable-next-line no-await-in-loop + response = await hook(response); + + if (is.plainObject(response)) { + if (emitter.retry(response) === false) { + reject(new Error('Retry limit reached.')); + } + return; + } + } + } catch (error) { + reject(error); + return; + } + if (options.json && response.body) { try { response.body = JSON.parse(response.body); diff --git a/source/index.js b/source/index.js index f3e4d3a20..a8f49aa98 100644 --- a/source/index.js +++ b/source/index.js @@ -28,7 +28,10 @@ const defaults = { 'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)` }, hooks: { - beforeRequest: [] + beforeRequest: [], + beforeRedirect: [], + beforeRetry: [], + afterResponse: [] }, decompress: true, throwHttpErrors: true, diff --git a/source/known-hook-events.js b/source/known-hook-events.js index d67dff280..2fa0381f5 100644 --- a/source/known-hook-events.js +++ b/source/known-hook-events.js @@ -1,3 +1,8 @@ 'use strict'; -module.exports = ['beforeRequest']; +module.exports = [ + 'beforeRequest', + 'beforeRedirect', + 'beforeRetry', + 'afterResponse' +]; diff --git a/source/normalize-arguments.js b/source/normalize-arguments.js index 5e73f536d..158ff53de 100644 --- a/source/normalize-arguments.js +++ b/source/normalize-arguments.js @@ -24,7 +24,7 @@ const preNormalize = (options, defaults) => { options.baseUrl += '/'; } - if (options.stream && options.json) { + if (options.stream) { options.json = false; } @@ -205,27 +205,33 @@ module.exports = (url, options, defaults) => { const {retries} = options.retry; options.retry.retries = (iteration, error) => { - if (iteration > retries || (!isRetryOnNetworkErrorAllowed(error) && (!options.retry.methods.has(error.method) || !options.retry.statusCodes.has(error.statusCode)))) { + if (iteration > retries) { return 0; } - if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) { - let after = Number(error.headers['retry-after']); - if (is.nan(after)) { - after = Date.parse(error.headers['retry-after']) - Date.now(); - } else { - after *= 1000; - } - - if (after > options.retry.maxRetryAfter) { + if (error !== null) { + if (!isRetryOnNetworkErrorAllowed(error) && (!options.retry.methods.has(error.method) || !options.retry.statusCodes.has(error.statusCode))) { return 0; } - return after; - } + if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) { + let after = Number(error.headers['retry-after']); + if (is.nan(after)) { + after = Date.parse(error.headers['retry-after']) - Date.now(); + } else { + after *= 1000; + } - if (error.statusCode === 413) { - return 0; + if (after > options.retry.maxRetryAfter) { + return 0; + } + + return after; + } + + if (error.statusCode === 413) { + return 0; + } } const noise = Math.random() * 100; diff --git a/source/request-as-event-emitter.js b/source/request-as-event-emitter.js index 5884dfd68..e752b8cf0 100644 --- a/source/request-as-event-emitter.js +++ b/source/request-as-event-emitter.js @@ -124,6 +124,11 @@ module.exports = (options, input) => { ...urlToOptions(redirectURL) }; + for (const hook of options.hooks.beforeRedirect) { + // eslint-disable-next-line no-await-in-loop + await hook(redirectOpts); + } + emitter.emit('redirect', response, redirectOpts); await get(redirectOpts); @@ -219,6 +224,17 @@ module.exports = (options, input) => { emitter.retry = error => { let backoff; + + if (is.plainObject(error)) { + // Retry with updated options + options = { + ...options, + ...error + }; + + error = null; + } + try { backoff = options.retry.retries(++retryTries, error); } catch (error2) { @@ -227,8 +243,21 @@ module.exports = (options, input) => { } if (backoff) { - retryCount++; - setTimeout(get, backoff, {...options, forceRefresh: true}); + const retry = async options => { + try { + for (const hook of options.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(options, error, retryCount); + } + + retryCount++; + await get(options); + } catch (error) { + emitter.emit('error', error); + } + }; + + setTimeout(retry, backoff, {...options, forceRefresh: true}); return true; } diff --git a/test/hooks.js b/test/hooks.js index b3d661483..0e8ec07ca 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -3,6 +3,8 @@ import delay from 'delay'; import {createServer} from './helpers/server'; import got from '..'; +const errorString = 'oops'; +const error = new Error(errorString); let s; test.before('setup', async () => { @@ -12,7 +14,32 @@ test.before('setup', async () => { response.write(JSON.stringify(request.headers)); response.end(); }; + s.on('/', echoHeaders); + s.on('/redirect', (request, response) => { + response.statusCode = 302; + response.setHeader('location', '/'); + response.end(); + }); + s.on('/retry', (request, response) => { + if (request.headers.foo) { + response.statusCode = 302; + response.setHeader('location', '/'); + response.end(); + } + + response.statusCode = 500; + response.end(); + }); + + s.on('/401', (request, response) => { + if (request.headers.token !== 'unicorn') { + response.statusCode = 401; + } + + response.end(); + }); + await s.listen(s.port); }); @@ -20,86 +47,209 @@ test.after('cleanup', async () => { await s.close(); }); -test('beforeRequest receives normalized options', async t => { - await got( - s.url, - { - json: true, - hooks: { - beforeRequest: [ - options => { - t.is(options.path, '/'); - t.is(options.hostname, 'localhost'); - } - ] - } +test('async hooks', async t => { + const response = await got(s.url, { + json: true, + hooks: { + beforeRequest: [ + async options => { + await delay(100); + options.headers.foo = 'bar'; + } + ] + } + }); + t.is(response.body.foo, 'bar'); +}); + +test('catches thrown errors', async t => { + await t.throwsAsync(() => got(s.url, { + hooks: { + beforeRequest: [ + () => { + throw error; + } + ] + } + }), errorString); +}); + +test('catches promise rejections', async t => { + await t.throwsAsync(() => got(s.url, { + hooks: { + beforeRequest: [ + () => Promise.reject(error) + ] + } + }), errorString); +}); + +test('catches beforeRequest errors', async t => { + await t.throwsAsync(() => got(s.url, { + hooks: { + beforeRequest: [() => Promise.reject(error)] } - ); + }), errorString); +}); + +test('catches beforeRedirect errors', async t => { + await t.throwsAsync(() => got(`${s.url}/redirect`, { + hooks: { + beforeRedirect: [() => Promise.reject(error)] + } + }), errorString); +}); + +test('catches beforeRetry errors', async t => { + await t.throwsAsync(() => got(`${s.url}/retry`, { + hooks: { + beforeRetry: [() => Promise.reject(error)] + } + }), errorString); +}); + +test('catches afterResponse errors', async t => { + await t.throwsAsync(() => got(s.url, { + hooks: { + afterResponse: [() => Promise.reject(error)] + } + }), errorString); +}); + +test('beforeRequest', async t => { + await got(s.url, { + json: true, + hooks: { + beforeRequest: [ + options => { + t.is(options.path, '/'); + t.is(options.hostname, 'localhost'); + } + ] + } + }); }); test('beforeRequest allows modifications', async t => { - const response = await got( - s.url, - { - json: true, - hooks: { - beforeRequest: [ - options => { - options.headers.foo = 'bar'; - } - ] - } + const response = await got(s.url, { + json: true, + hooks: { + beforeRequest: [ + options => { + options.headers.foo = 'bar'; + } + ] } - ); + }); t.is(response.body.foo, 'bar'); }); -test('beforeRequest awaits async function', async t => { - const response = await got( - s.url, - { - json: true, - hooks: { - beforeRequest: [ - async options => { - await delay(100); - options.headers.foo = 'bar'; - } - ] - } +test('beforeRedirect', async t => { + await got(`${s.url}/redirect`, { + json: true, + hooks: { + beforeRedirect: [ + options => { + t.is(options.path, '/'); + t.is(options.hostname, 'localhost'); + } + ] } - ); + }); +}); + +test('beforeRedirect allows modifications', async t => { + const response = await got(`${s.url}/redirect`, { + json: true, + hooks: { + beforeRedirect: [ + options => { + options.headers.foo = 'bar'; + } + ] + } + }); t.is(response.body.foo, 'bar'); }); -test('beforeRequest rejects when beforeRequest throws', async t => { - await t.throwsAsync( - () => got(s.url, { - hooks: { - beforeRequest: [ - () => { - throw new Error('oops'); +test('beforeRetry', async t => { + await got(`${s.url}/retry`, { + json: true, + retry: 1, + throwHttpErrors: false, + hooks: { + beforeRetry: [ + options => { + t.is(options.hostname, 'localhost'); + } + ] + } + }); +}); + +test('beforeRetry allows modifications', async t => { + const response = await got(`${s.url}/retry`, { + json: true, + hooks: { + beforeRetry: [ + options => { + options.headers.foo = 'bar'; + } + ] + } + }); + t.is(response.body.foo, 'bar'); +}); + +test('afterResponse', async t => { + await got(`${s.url}`, { + json: true, + hooks: { + afterResponse: [ + response => { + t.is(typeof response.body, 'string'); + + return response; + } + ] + } + }); +}); + +test('afterResponse allows modifications', async t => { + const response = await got(`${s.url}`, { + json: true, + hooks: { + afterResponse: [ + response => { + response.body = '{"hello": "world"}'; + + return response; + } + ] + } + }); + t.is(response.body.hello, 'world'); +}); + +test('afterResponse allows to retry', async t => { + const response = await got(`${s.url}/401`, { + json: true, + hooks: { + afterResponse: [ + response => { + if (response.statusCode === 401) { + return { + headers: { + token: 'unicorn' + } + }; } - ] - } - }), - { - instanceOf: Error, - message: 'oops' - } - ); -}); - -test('beforeRequest rejects when beforeRequest rejects', async t => { - await t.throwsAsync( - () => got(s.url, { - hooks: { - beforeRequest: [() => Promise.reject(new Error('oops'))] - } - }), - { - instanceOf: Error, - message: 'oops' - } - ); + + return response; + } + ] + } + }); + t.is(response.statusCode, 200); });