You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

index.js 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. var url = require("url");
  2. var URL = url.URL;
  3. var http = require("http");
  4. var https = require("https");
  5. var assert = require("assert");
  6. var Writable = require("stream").Writable;
  7. var debug = require("debug")("follow-redirects");
  8. // RFC7231§4.2.1: Of the request methods defined by this specification,
  9. // the GET, HEAD, OPTIONS, and TRACE methods are defined to be safe.
  10. var SAFE_METHODS = { GET: true, HEAD: true, OPTIONS: true, TRACE: true };
  11. // Create handlers that pass events from native requests
  12. var eventHandlers = Object.create(null);
  13. ["abort", "aborted", "error", "socket", "timeout"].forEach(function (event) {
  14. eventHandlers[event] = function (arg) {
  15. this._redirectable.emit(event, arg);
  16. };
  17. });
  18. // An HTTP(S) request that can be redirected
  19. function RedirectableRequest(options, responseCallback) {
  20. // Initialize the request
  21. Writable.call(this);
  22. options.headers = options.headers || {};
  23. this._options = options;
  24. this._ended = false;
  25. this._ending = false;
  26. this._redirectCount = 0;
  27. this._redirects = [];
  28. this._requestBodyLength = 0;
  29. this._requestBodyBuffers = [];
  30. // Since http.request treats host as an alias of hostname,
  31. // but the url module interprets host as hostname plus port,
  32. // eliminate the host property to avoid confusion.
  33. if (options.host) {
  34. // Use hostname if set, because it has precedence
  35. if (!options.hostname) {
  36. options.hostname = options.host;
  37. }
  38. delete options.host;
  39. }
  40. // Attach a callback if passed
  41. if (responseCallback) {
  42. this.on("response", responseCallback);
  43. }
  44. // React to responses of native requests
  45. var self = this;
  46. this._onNativeResponse = function (response) {
  47. self._processResponse(response);
  48. };
  49. // Complete the URL object when necessary
  50. if (!options.pathname && options.path) {
  51. var searchPos = options.path.indexOf("?");
  52. if (searchPos < 0) {
  53. options.pathname = options.path;
  54. }
  55. else {
  56. options.pathname = options.path.substring(0, searchPos);
  57. options.search = options.path.substring(searchPos);
  58. }
  59. }
  60. // Perform the first request
  61. this._performRequest();
  62. }
  63. RedirectableRequest.prototype = Object.create(Writable.prototype);
  64. // Writes buffered data to the current native request
  65. RedirectableRequest.prototype.write = function (data, encoding, callback) {
  66. // Writing is not allowed if end has been called
  67. if (this._ending) {
  68. throw new Error("write after end");
  69. }
  70. // Validate input and shift parameters if necessary
  71. if (!(typeof data === "string" || typeof data === "object" && ("length" in data))) {
  72. throw new Error("data should be a string, Buffer or Uint8Array");
  73. }
  74. if (typeof encoding === "function") {
  75. callback = encoding;
  76. encoding = null;
  77. }
  78. // Ignore empty buffers, since writing them doesn't invoke the callback
  79. // https://github.com/nodejs/node/issues/22066
  80. if (data.length === 0) {
  81. if (callback) {
  82. callback();
  83. }
  84. return;
  85. }
  86. // Only write when we don't exceed the maximum body length
  87. if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
  88. this._requestBodyLength += data.length;
  89. this._requestBodyBuffers.push({ data: data, encoding: encoding });
  90. this._currentRequest.write(data, encoding, callback);
  91. }
  92. // Error when we exceed the maximum body length
  93. else {
  94. this.emit("error", new Error("Request body larger than maxBodyLength limit"));
  95. this.abort();
  96. }
  97. };
  98. // Ends the current native request
  99. RedirectableRequest.prototype.end = function (data, encoding, callback) {
  100. // Shift parameters if necessary
  101. if (typeof data === "function") {
  102. callback = data;
  103. data = encoding = null;
  104. }
  105. else if (typeof encoding === "function") {
  106. callback = encoding;
  107. encoding = null;
  108. }
  109. // Write data if needed and end
  110. if (!data) {
  111. this._ended = this._ending = true;
  112. this._currentRequest.end(null, null, callback);
  113. }
  114. else {
  115. var self = this;
  116. var currentRequest = this._currentRequest;
  117. this.write(data, encoding, function () {
  118. self._ended = true;
  119. currentRequest.end(null, null, callback);
  120. });
  121. this._ending = true;
  122. }
  123. };
  124. // Sets a header value on the current native request
  125. RedirectableRequest.prototype.setHeader = function (name, value) {
  126. this._options.headers[name] = value;
  127. this._currentRequest.setHeader(name, value);
  128. };
  129. // Clears a header value on the current native request
  130. RedirectableRequest.prototype.removeHeader = function (name) {
  131. delete this._options.headers[name];
  132. this._currentRequest.removeHeader(name);
  133. };
  134. // Global timeout for all underlying requests
  135. RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
  136. if (callback) {
  137. this.once("timeout", callback);
  138. }
  139. if (this.socket) {
  140. startTimer(this, msecs);
  141. }
  142. else {
  143. var self = this;
  144. this._currentRequest.once("socket", function () {
  145. startTimer(self, msecs);
  146. });
  147. }
  148. this.once("response", clearTimer);
  149. this.once("error", clearTimer);
  150. return this;
  151. };
  152. function startTimer(request, msecs) {
  153. clearTimeout(request._timeout);
  154. request._timeout = setTimeout(function () {
  155. request.emit("timeout");
  156. }, msecs);
  157. }
  158. function clearTimer() {
  159. clearTimeout(this._timeout);
  160. }
  161. // Proxy all other public ClientRequest methods
  162. [
  163. "abort", "flushHeaders", "getHeader",
  164. "setNoDelay", "setSocketKeepAlive",
  165. ].forEach(function (method) {
  166. RedirectableRequest.prototype[method] = function (a, b) {
  167. return this._currentRequest[method](a, b);
  168. };
  169. });
  170. // Proxy all public ClientRequest properties
  171. ["aborted", "connection", "socket"].forEach(function (property) {
  172. Object.defineProperty(RedirectableRequest.prototype, property, {
  173. get: function () { return this._currentRequest[property]; },
  174. });
  175. });
  176. // Executes the next native request (initial or redirect)
  177. RedirectableRequest.prototype._performRequest = function () {
  178. // Load the native protocol
  179. var protocol = this._options.protocol;
  180. var nativeProtocol = this._options.nativeProtocols[protocol];
  181. if (!nativeProtocol) {
  182. this.emit("error", new Error("Unsupported protocol " + protocol));
  183. return;
  184. }
  185. // If specified, use the agent corresponding to the protocol
  186. // (HTTP and HTTPS use different types of agents)
  187. if (this._options.agents) {
  188. var scheme = protocol.substr(0, protocol.length - 1);
  189. this._options.agent = this._options.agents[scheme];
  190. }
  191. // Create the native request
  192. var request = this._currentRequest =
  193. nativeProtocol.request(this._options, this._onNativeResponse);
  194. this._currentUrl = url.format(this._options);
  195. // Set up event handlers
  196. request._redirectable = this;
  197. for (var event in eventHandlers) {
  198. /* istanbul ignore else */
  199. if (event) {
  200. request.on(event, eventHandlers[event]);
  201. }
  202. }
  203. // End a redirected request
  204. // (The first request must be ended explicitly with RedirectableRequest#end)
  205. if (this._isRedirect) {
  206. // Write the request entity and end.
  207. var i = 0;
  208. var self = this;
  209. var buffers = this._requestBodyBuffers;
  210. (function writeNext(error) {
  211. // Only write if this request has not been redirected yet
  212. /* istanbul ignore else */
  213. if (request === self._currentRequest) {
  214. // Report any write errors
  215. /* istanbul ignore if */
  216. if (error) {
  217. self.emit("error", error);
  218. }
  219. // Write the next buffer if there are still left
  220. else if (i < buffers.length) {
  221. var buffer = buffers[i++];
  222. /* istanbul ignore else */
  223. if (!request.finished) {
  224. request.write(buffer.data, buffer.encoding, writeNext);
  225. }
  226. }
  227. // End the request if `end` has been called on us
  228. else if (self._ended) {
  229. request.end();
  230. }
  231. }
  232. }());
  233. }
  234. };
  235. // Processes a response from the current native request
  236. RedirectableRequest.prototype._processResponse = function (response) {
  237. // Store the redirected response
  238. if (this._options.trackRedirects) {
  239. this._redirects.push({
  240. url: this._currentUrl,
  241. headers: response.headers,
  242. statusCode: response.statusCode,
  243. });
  244. }
  245. // RFC7231§6.4: The 3xx (Redirection) class of status code indicates
  246. // that further action needs to be taken by the user agent in order to
  247. // fulfill the request. If a Location header field is provided,
  248. // the user agent MAY automatically redirect its request to the URI
  249. // referenced by the Location field value,
  250. // even if the specific status code is not understood.
  251. var location = response.headers.location;
  252. if (location && this._options.followRedirects !== false &&
  253. response.statusCode >= 300 && response.statusCode < 400) {
  254. // Abort the current request
  255. this._currentRequest.removeAllListeners();
  256. this._currentRequest.on("error", noop);
  257. this._currentRequest.abort();
  258. // RFC7231§6.4: A client SHOULD detect and intervene
  259. // in cyclical redirections (i.e., "infinite" redirection loops).
  260. if (++this._redirectCount > this._options.maxRedirects) {
  261. this.emit("error", new Error("Max redirects exceeded."));
  262. return;
  263. }
  264. // RFC7231§6.4: Automatic redirection needs to done with
  265. // care for methods not known to be safe […],
  266. // since the user might not wish to redirect an unsafe request.
  267. // RFC7231§6.4.7: The 307 (Temporary Redirect) status code indicates
  268. // that the target resource resides temporarily under a different URI
  269. // and the user agent MUST NOT change the request method
  270. // if it performs an automatic redirection to that URI.
  271. var header;
  272. var headers = this._options.headers;
  273. if (response.statusCode !== 307 && !(this._options.method in SAFE_METHODS)) {
  274. this._options.method = "GET";
  275. // Drop a possible entity and headers related to it
  276. this._requestBodyBuffers = [];
  277. for (header in headers) {
  278. if (/^content-/i.test(header)) {
  279. delete headers[header];
  280. }
  281. }
  282. }
  283. // Drop the Host header, as the redirect might lead to a different host
  284. if (!this._isRedirect) {
  285. for (header in headers) {
  286. if (/^host$/i.test(header)) {
  287. delete headers[header];
  288. }
  289. }
  290. }
  291. // Perform the redirected request
  292. var redirectUrl = url.resolve(this._currentUrl, location);
  293. debug("redirecting to", redirectUrl);
  294. Object.assign(this._options, url.parse(redirectUrl));
  295. this._isRedirect = true;
  296. this._performRequest();
  297. // Discard the remainder of the response to avoid waiting for data
  298. response.destroy();
  299. }
  300. else {
  301. // The response is not a redirect; return it as-is
  302. response.responseUrl = this._currentUrl;
  303. response.redirects = this._redirects;
  304. this.emit("response", response);
  305. // Clean up
  306. this._requestBodyBuffers = [];
  307. }
  308. };
  309. // Wraps the key/value object of protocols with redirect functionality
  310. function wrap(protocols) {
  311. // Default settings
  312. var exports = {
  313. maxRedirects: 21,
  314. maxBodyLength: 10 * 1024 * 1024,
  315. };
  316. // Wrap each protocol
  317. var nativeProtocols = {};
  318. Object.keys(protocols).forEach(function (scheme) {
  319. var protocol = scheme + ":";
  320. var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
  321. var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
  322. // Executes a request, following redirects
  323. wrappedProtocol.request = function (input, options, callback) {
  324. // Parse parameters
  325. if (typeof input === "string") {
  326. var urlStr = input;
  327. try {
  328. input = urlToOptions(new URL(urlStr));
  329. }
  330. catch (err) {
  331. /* istanbul ignore next */
  332. input = url.parse(urlStr);
  333. }
  334. }
  335. else if (URL && (input instanceof URL)) {
  336. input = urlToOptions(input);
  337. }
  338. else {
  339. callback = options;
  340. options = input;
  341. input = { protocol: protocol };
  342. }
  343. if (typeof options === "function") {
  344. callback = options;
  345. options = null;
  346. }
  347. // Set defaults
  348. options = Object.assign({
  349. maxRedirects: exports.maxRedirects,
  350. maxBodyLength: exports.maxBodyLength,
  351. }, input, options);
  352. options.nativeProtocols = nativeProtocols;
  353. assert.equal(options.protocol, protocol, "protocol mismatch");
  354. debug("options", options);
  355. return new RedirectableRequest(options, callback);
  356. };
  357. // Executes a GET request, following redirects
  358. wrappedProtocol.get = function (input, options, callback) {
  359. var request = wrappedProtocol.request(input, options, callback);
  360. request.end();
  361. return request;
  362. };
  363. });
  364. return exports;
  365. }
  366. /* istanbul ignore next */
  367. function noop() { /* empty */ }
  368. // from https://github.com/nodejs/node/blob/master/lib/internal/url.js
  369. function urlToOptions(urlObject) {
  370. var options = {
  371. protocol: urlObject.protocol,
  372. hostname: urlObject.hostname.startsWith("[") ?
  373. /* istanbul ignore next */
  374. urlObject.hostname.slice(1, -1) :
  375. urlObject.hostname,
  376. hash: urlObject.hash,
  377. search: urlObject.search,
  378. pathname: urlObject.pathname,
  379. path: urlObject.pathname + urlObject.search,
  380. href: urlObject.href,
  381. };
  382. if (urlObject.port !== "") {
  383. options.port = Number(urlObject.port);
  384. }
  385. return options;
  386. }
  387. // Exports
  388. module.exports = wrap({ http: http, https: https });
  389. module.exports.wrap = wrap;