You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

index.js 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479
  1. 'use strict';
  2. const EventEmitter = require('events');
  3. const http = require('http');
  4. const https = require('https');
  5. const PassThrough = require('stream').PassThrough;
  6. const urlLib = require('url');
  7. const querystring = require('querystring');
  8. const duplexer3 = require('duplexer3');
  9. const isStream = require('is-stream');
  10. const getStream = require('get-stream');
  11. const timedOut = require('timed-out');
  12. const urlParseLax = require('url-parse-lax');
  13. const urlToOptions = require('url-to-options');
  14. const lowercaseKeys = require('lowercase-keys');
  15. const decompressResponse = require('decompress-response');
  16. const isRetryAllowed = require('is-retry-allowed');
  17. const Buffer = require('safe-buffer').Buffer;
  18. const isURL = require('isurl');
  19. const isPlainObj = require('is-plain-obj');
  20. const PCancelable = require('p-cancelable');
  21. const pTimeout = require('p-timeout');
  22. const pkg = require('./package');
  23. const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
  24. const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
  25. function requestAsEventEmitter(opts) {
  26. opts = opts || {};
  27. const ee = new EventEmitter();
  28. const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
  29. const redirects = [];
  30. let retryCount = 0;
  31. let redirectUrl;
  32. const get = opts => {
  33. if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
  34. ee.emit('error', new got.UnsupportedProtocolError(opts));
  35. return;
  36. }
  37. let fn = opts.protocol === 'https:' ? https : http;
  38. if (opts.useElectronNet && process.versions.electron) {
  39. const electron = require('electron');
  40. fn = electron.net || electron.remote.net;
  41. }
  42. const req = fn.request(opts, res => {
  43. const statusCode = res.statusCode;
  44. res.url = redirectUrl || requestUrl;
  45. res.requestUrl = requestUrl;
  46. const followRedirect = opts.followRedirect && 'location' in res.headers;
  47. const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
  48. const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
  49. if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) {
  50. res.resume();
  51. if (statusCode === 303) {
  52. // Server responded with "see other", indicating that the resource exists at another location,
  53. // and the client should request it from that location via GET or HEAD.
  54. opts.method = 'GET';
  55. }
  56. if (redirects.length >= 10) {
  57. ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res);
  58. return;
  59. }
  60. const bufferString = Buffer.from(res.headers.location, 'binary').toString();
  61. redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString);
  62. redirects.push(redirectUrl);
  63. const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl));
  64. ee.emit('redirect', res, redirectOpts);
  65. get(redirectOpts);
  66. return;
  67. }
  68. setImmediate(() => {
  69. const response = opts.decompress === true &&
  70. typeof decompressResponse === 'function' &&
  71. req.method !== 'HEAD' ? decompressResponse(res) : res;
  72. if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
  73. opts.encoding = null;
  74. }
  75. response.redirectUrls = redirects;
  76. ee.emit('response', response);
  77. });
  78. });
  79. req.once('error', err => {
  80. const backoff = opts.retries(++retryCount, err);
  81. if (backoff) {
  82. setTimeout(get, backoff, opts);
  83. return;
  84. }
  85. ee.emit('error', new got.RequestError(err, opts));
  86. });
  87. if (opts.gotTimeout) {
  88. timedOut(req, opts.gotTimeout);
  89. }
  90. setImmediate(() => {
  91. ee.emit('request', req);
  92. });
  93. };
  94. setImmediate(() => {
  95. get(opts);
  96. });
  97. return ee;
  98. }
  99. function asPromise(opts) {
  100. const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ?
  101. pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
  102. requestPromise;
  103. return timeoutFn(new PCancelable((onCancel, resolve, reject) => {
  104. const ee = requestAsEventEmitter(opts);
  105. let cancelOnRequest = false;
  106. onCancel(() => {
  107. cancelOnRequest = true;
  108. });
  109. ee.on('request', req => {
  110. if (cancelOnRequest) {
  111. req.abort();
  112. }
  113. onCancel(() => {
  114. req.abort();
  115. });
  116. if (isStream(opts.body)) {
  117. opts.body.pipe(req);
  118. opts.body = undefined;
  119. return;
  120. }
  121. req.end(opts.body);
  122. });
  123. ee.on('response', res => {
  124. const stream = opts.encoding === null ? getStream.buffer(res) : getStream(res, opts);
  125. stream
  126. .catch(err => reject(new got.ReadError(err, opts)))
  127. .then(data => {
  128. const statusCode = res.statusCode;
  129. const limitStatusCode = opts.followRedirect ? 299 : 399;
  130. res.body = data;
  131. if (opts.json && res.body) {
  132. try {
  133. res.body = JSON.parse(res.body);
  134. } catch (e) {
  135. if (statusCode >= 200 && statusCode < 300) {
  136. throw new got.ParseError(e, statusCode, opts, data);
  137. }
  138. }
  139. }
  140. if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
  141. throw new got.HTTPError(statusCode, res.headers, opts);
  142. }
  143. resolve(res);
  144. })
  145. .catch(err => {
  146. Object.defineProperty(err, 'response', {value: res});
  147. reject(err);
  148. });
  149. });
  150. ee.on('error', reject);
  151. }));
  152. }
  153. function asStream(opts) {
  154. const input = new PassThrough();
  155. const output = new PassThrough();
  156. const proxy = duplexer3(input, output);
  157. let timeout;
  158. if (opts.gotTimeout && opts.gotTimeout.request) {
  159. timeout = setTimeout(() => {
  160. proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts));
  161. }, opts.gotTimeout.request);
  162. }
  163. if (opts.json) {
  164. throw new Error('got can not be used as stream when options.json is used');
  165. }
  166. if (opts.body) {
  167. proxy.write = () => {
  168. throw new Error('got\'s stream is not writable when options.body is used');
  169. };
  170. }
  171. const ee = requestAsEventEmitter(opts);
  172. ee.on('request', req => {
  173. proxy.emit('request', req);
  174. if (isStream(opts.body)) {
  175. opts.body.pipe(req);
  176. return;
  177. }
  178. if (opts.body) {
  179. req.end(opts.body);
  180. return;
  181. }
  182. if (opts.method === 'POST' || opts.method === 'PUT' || opts.method === 'PATCH') {
  183. input.pipe(req);
  184. return;
  185. }
  186. req.end();
  187. });
  188. ee.on('response', res => {
  189. clearTimeout(timeout);
  190. const statusCode = res.statusCode;
  191. res.pipe(output);
  192. if (statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
  193. proxy.emit('error', new got.HTTPError(statusCode, res.headers, opts), null, res);
  194. return;
  195. }
  196. proxy.emit('response', res);
  197. });
  198. ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
  199. ee.on('error', proxy.emit.bind(proxy, 'error'));
  200. return proxy;
  201. }
  202. function normalizeArguments(url, opts) {
  203. if (typeof url !== 'string' && typeof url !== 'object') {
  204. throw new TypeError(`Parameter \`url\` must be a string or object, not ${typeof url}`);
  205. } else if (typeof url === 'string') {
  206. url = url.replace(/^unix:/, 'http://$&');
  207. url = urlParseLax(url);
  208. } else if (isURL.lenient(url)) {
  209. url = urlToOptions(url);
  210. }
  211. if (url.auth) {
  212. throw new Error('Basic authentication must be done with auth option');
  213. }
  214. opts = Object.assign(
  215. {
  216. path: '',
  217. retries: 2,
  218. decompress: true,
  219. useElectronNet: true
  220. },
  221. url,
  222. {
  223. protocol: url.protocol || 'http:' // Override both null/undefined with default protocol
  224. },
  225. opts
  226. );
  227. opts.headers = Object.assign({
  228. 'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`,
  229. 'accept-encoding': 'gzip,deflate'
  230. }, lowercaseKeys(opts.headers));
  231. const query = opts.query;
  232. if (query) {
  233. if (typeof query !== 'string') {
  234. opts.query = querystring.stringify(query);
  235. }
  236. opts.path = `${opts.path.split('?')[0]}?${opts.query}`;
  237. delete opts.query;
  238. }
  239. if (opts.json && opts.headers.accept === undefined) {
  240. opts.headers.accept = 'application/json';
  241. }
  242. const body = opts.body;
  243. if (body !== null && body !== undefined) {
  244. const headers = opts.headers;
  245. if (!isStream(body) && typeof body !== 'string' && !Buffer.isBuffer(body) && !(opts.form || opts.json)) {
  246. throw new TypeError('options.body must be a ReadableStream, string, Buffer or plain Object');
  247. }
  248. const canBodyBeStringified = isPlainObj(body) || Array.isArray(body);
  249. if ((opts.form || opts.json) && !canBodyBeStringified) {
  250. throw new TypeError('options.body must be a plain Object or Array when options.form or options.json is used');
  251. }
  252. if (isStream(body) && typeof body.getBoundary === 'function') {
  253. // Special case for https://github.com/form-data/form-data
  254. headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
  255. } else if (opts.form && canBodyBeStringified) {
  256. headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
  257. opts.body = querystring.stringify(body);
  258. } else if (opts.json && canBodyBeStringified) {
  259. headers['content-type'] = headers['content-type'] || 'application/json';
  260. opts.body = JSON.stringify(body);
  261. }
  262. if (headers['content-length'] === undefined && headers['transfer-encoding'] === undefined && !isStream(body)) {
  263. const length = typeof opts.body === 'string' ? Buffer.byteLength(opts.body) : opts.body.length;
  264. headers['content-length'] = length;
  265. }
  266. opts.method = (opts.method || 'POST').toUpperCase();
  267. } else {
  268. opts.method = (opts.method || 'GET').toUpperCase();
  269. }
  270. if (opts.hostname === 'unix') {
  271. const matches = /(.+?):(.+)/.exec(opts.path);
  272. if (matches) {
  273. opts.socketPath = matches[1];
  274. opts.path = matches[2];
  275. opts.host = null;
  276. }
  277. }
  278. if (typeof opts.retries !== 'function') {
  279. const retries = opts.retries;
  280. opts.retries = (iter, err) => {
  281. if (iter > retries || !isRetryAllowed(err)) {
  282. return 0;
  283. }
  284. const noise = Math.random() * 100;
  285. return ((1 << iter) * 1000) + noise;
  286. };
  287. }
  288. if (opts.followRedirect === undefined) {
  289. opts.followRedirect = true;
  290. }
  291. if (opts.timeout) {
  292. if (typeof opts.timeout === 'number') {
  293. opts.gotTimeout = {request: opts.timeout};
  294. } else {
  295. opts.gotTimeout = opts.timeout;
  296. }
  297. delete opts.timeout;
  298. }
  299. return opts;
  300. }
  301. function got(url, opts) {
  302. try {
  303. return asPromise(normalizeArguments(url, opts));
  304. } catch (err) {
  305. return Promise.reject(err);
  306. }
  307. }
  308. got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
  309. const methods = [
  310. 'get',
  311. 'post',
  312. 'put',
  313. 'patch',
  314. 'head',
  315. 'delete'
  316. ];
  317. for (const method of methods) {
  318. got[method] = (url, opts) => got(url, Object.assign({}, opts, {method}));
  319. got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method}));
  320. }
  321. class StdError extends Error {
  322. constructor(message, error, opts) {
  323. super(message);
  324. this.name = 'StdError';
  325. if (error.code !== undefined) {
  326. this.code = error.code;
  327. }
  328. Object.assign(this, {
  329. host: opts.host,
  330. hostname: opts.hostname,
  331. method: opts.method,
  332. path: opts.path,
  333. protocol: opts.protocol,
  334. url: opts.href
  335. });
  336. }
  337. }
  338. got.RequestError = class extends StdError {
  339. constructor(error, opts) {
  340. super(error.message, error, opts);
  341. this.name = 'RequestError';
  342. }
  343. };
  344. got.ReadError = class extends StdError {
  345. constructor(error, opts) {
  346. super(error.message, error, opts);
  347. this.name = 'ReadError';
  348. }
  349. };
  350. got.ParseError = class extends StdError {
  351. constructor(error, statusCode, opts, data) {
  352. super(`${error.message} in "${urlLib.format(opts)}": \n${data.slice(0, 77)}...`, error, opts);
  353. this.name = 'ParseError';
  354. this.statusCode = statusCode;
  355. this.statusMessage = http.STATUS_CODES[this.statusCode];
  356. }
  357. };
  358. got.HTTPError = class extends StdError {
  359. constructor(statusCode, headers, opts) {
  360. const statusMessage = http.STATUS_CODES[statusCode];
  361. super(`Response code ${statusCode} (${statusMessage})`, {}, opts);
  362. this.name = 'HTTPError';
  363. this.statusCode = statusCode;
  364. this.statusMessage = statusMessage;
  365. this.headers = headers;
  366. }
  367. };
  368. got.MaxRedirectsError = class extends StdError {
  369. constructor(statusCode, redirectUrls, opts) {
  370. super('Redirected 10 times. Aborting.', {}, opts);
  371. this.name = 'MaxRedirectsError';
  372. this.statusCode = statusCode;
  373. this.statusMessage = http.STATUS_CODES[this.statusCode];
  374. this.redirectUrls = redirectUrls;
  375. }
  376. };
  377. got.UnsupportedProtocolError = class extends StdError {
  378. constructor(opts) {
  379. super(`Unsupported protocol "${opts.protocol}"`, {}, opts);
  380. this.name = 'UnsupportedProtocolError';
  381. }
  382. };
  383. module.exports = got;