|
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034 |
- """
- requests.models
- ~~~~~~~~~~~~~~~
-
- This module contains the primary objects that power Requests.
- """
-
- import datetime
-
- # Import encoding now, to avoid implicit import later.
- # Implicit import within threads may cause LookupError when standard library is in a ZIP,
- # such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
- import encodings.idna # noqa: F401
- from io import UnsupportedOperation
-
- from urllib3.exceptions import (
- DecodeError,
- LocationParseError,
- ProtocolError,
- ReadTimeoutError,
- SSLError,
- )
- from urllib3.fields import RequestField
- from urllib3.filepost import encode_multipart_formdata
- from urllib3.util import parse_url
-
- from ._internal_utils import to_native_string, unicode_is_ascii
- from .auth import HTTPBasicAuth
- from .compat import (
- Callable,
- JSONDecodeError,
- Mapping,
- basestring,
- builtin_str,
- chardet,
- cookielib,
- )
- from .compat import json as complexjson
- from .compat import urlencode, urlsplit, urlunparse
- from .cookies import _copy_cookie_jar, cookiejar_from_dict, get_cookie_header
- from .exceptions import (
- ChunkedEncodingError,
- ConnectionError,
- ContentDecodingError,
- HTTPError,
- InvalidJSONError,
- InvalidURL,
- )
- from .exceptions import JSONDecodeError as RequestsJSONDecodeError
- from .exceptions import MissingSchema
- from .exceptions import SSLError as RequestsSSLError
- from .exceptions import StreamConsumedError
- from .hooks import default_hooks
- from .status_codes import codes
- from .structures import CaseInsensitiveDict
- from .utils import (
- check_header_validity,
- get_auth_from_url,
- guess_filename,
- guess_json_utf,
- iter_slices,
- parse_header_links,
- requote_uri,
- stream_decode_response_unicode,
- super_len,
- to_key_val_list,
- )
-
- #: The set of HTTP status codes that indicate an automatically
- #: processable redirect.
- REDIRECT_STATI = (
- codes.moved, # 301
- codes.found, # 302
- codes.other, # 303
- codes.temporary_redirect, # 307
- codes.permanent_redirect, # 308
- )
-
- DEFAULT_REDIRECT_LIMIT = 30
- CONTENT_CHUNK_SIZE = 10 * 1024
- ITER_CHUNK_SIZE = 512
-
-
- class RequestEncodingMixin:
- @property
- def path_url(self):
- """Build the path URL to use."""
-
- url = []
-
- p = urlsplit(self.url)
-
- path = p.path
- if not path:
- path = "/"
-
- url.append(path)
-
- query = p.query
- if query:
- url.append("?")
- url.append(query)
-
- return "".join(url)
-
- @staticmethod
- def _encode_params(data):
- """Encode parameters in a piece of data.
-
- Will successfully encode parameters when passed as a dict or a list of
- 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
- if parameters are supplied as a dict.
- """
-
- if isinstance(data, (str, bytes)):
- return data
- elif hasattr(data, "read"):
- return data
- elif hasattr(data, "__iter__"):
- result = []
- for k, vs in to_key_val_list(data):
- if isinstance(vs, basestring) or not hasattr(vs, "__iter__"):
- vs = [vs]
- for v in vs:
- if v is not None:
- result.append(
- (
- k.encode("utf-8") if isinstance(k, str) else k,
- v.encode("utf-8") if isinstance(v, str) else v,
- )
- )
- return urlencode(result, doseq=True)
- else:
- return data
-
- @staticmethod
- def _encode_files(files, data):
- """Build the body for a multipart/form-data request.
-
- Will successfully encode files when passed as a dict or a list of
- tuples. Order is retained if data is a list of tuples but arbitrary
- if parameters are supplied as a dict.
- The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
- or 4-tuples (filename, fileobj, contentype, custom_headers).
- """
- if not files:
- raise ValueError("Files must be provided.")
- elif isinstance(data, basestring):
- raise ValueError("Data must not be a string.")
-
- new_fields = []
- fields = to_key_val_list(data or {})
- files = to_key_val_list(files or {})
-
- for field, val in fields:
- if isinstance(val, basestring) or not hasattr(val, "__iter__"):
- val = [val]
- for v in val:
- if v is not None:
- # Don't call str() on bytestrings: in Py3 it all goes wrong.
- if not isinstance(v, bytes):
- v = str(v)
-
- new_fields.append(
- (
- field.decode("utf-8")
- if isinstance(field, bytes)
- else field,
- v.encode("utf-8") if isinstance(v, str) else v,
- )
- )
-
- for (k, v) in files:
- # support for explicit filename
- ft = None
- fh = None
- if isinstance(v, (tuple, list)):
- if len(v) == 2:
- fn, fp = v
- elif len(v) == 3:
- fn, fp, ft = v
- else:
- fn, fp, ft, fh = v
- else:
- fn = guess_filename(v) or k
- fp = v
-
- if isinstance(fp, (str, bytes, bytearray)):
- fdata = fp
- elif hasattr(fp, "read"):
- fdata = fp.read()
- elif fp is None:
- continue
- else:
- fdata = fp
-
- rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
- rf.make_multipart(content_type=ft)
- new_fields.append(rf)
-
- body, content_type = encode_multipart_formdata(new_fields)
-
- return body, content_type
-
-
- class RequestHooksMixin:
- def register_hook(self, event, hook):
- """Properly register a hook."""
-
- if event not in self.hooks:
- raise ValueError(f'Unsupported event specified, with event name "{event}"')
-
- if isinstance(hook, Callable):
- self.hooks[event].append(hook)
- elif hasattr(hook, "__iter__"):
- self.hooks[event].extend(h for h in hook if isinstance(h, Callable))
-
- def deregister_hook(self, event, hook):
- """Deregister a previously registered hook.
- Returns True if the hook existed, False if not.
- """
-
- try:
- self.hooks[event].remove(hook)
- return True
- except ValueError:
- return False
-
-
- class Request(RequestHooksMixin):
- """A user-created :class:`Request <Request>` object.
-
- Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
-
- :param method: HTTP method to use.
- :param url: URL to send.
- :param headers: dictionary of headers to send.
- :param files: dictionary of {filename: fileobject} files to multipart upload.
- :param data: the body to attach to the request. If a dictionary or
- list of tuples ``[(key, value)]`` is provided, form-encoding will
- take place.
- :param json: json for the body to attach to the request (if files or data is not specified).
- :param params: URL parameters to append to the URL. If a dictionary or
- list of tuples ``[(key, value)]`` is provided, form-encoding will
- take place.
- :param auth: Auth handler or (user, pass) tuple.
- :param cookies: dictionary or CookieJar of cookies to attach to this request.
- :param hooks: dictionary of callback hooks, for internal usage.
-
- Usage::
-
- >>> import requests
- >>> req = requests.Request('GET', 'https://httpbin.org/get')
- >>> req.prepare()
- <PreparedRequest [GET]>
- """
-
- def __init__(
- self,
- method=None,
- url=None,
- headers=None,
- files=None,
- data=None,
- params=None,
- auth=None,
- cookies=None,
- hooks=None,
- json=None,
- ):
-
- # Default empty dicts for dict params.
- data = [] if data is None else data
- files = [] if files is None else files
- headers = {} if headers is None else headers
- params = {} if params is None else params
- hooks = {} if hooks is None else hooks
-
- self.hooks = default_hooks()
- for (k, v) in list(hooks.items()):
- self.register_hook(event=k, hook=v)
-
- self.method = method
- self.url = url
- self.headers = headers
- self.files = files
- self.data = data
- self.json = json
- self.params = params
- self.auth = auth
- self.cookies = cookies
-
- def __repr__(self):
- return f"<Request [{self.method}]>"
-
- def prepare(self):
- """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
- p = PreparedRequest()
- p.prepare(
- method=self.method,
- url=self.url,
- headers=self.headers,
- files=self.files,
- data=self.data,
- json=self.json,
- params=self.params,
- auth=self.auth,
- cookies=self.cookies,
- hooks=self.hooks,
- )
- return p
-
-
- class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
- """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
- containing the exact bytes that will be sent to the server.
-
- Instances are generated from a :class:`Request <Request>` object, and
- should not be instantiated manually; doing so may produce undesirable
- effects.
-
- Usage::
-
- >>> import requests
- >>> req = requests.Request('GET', 'https://httpbin.org/get')
- >>> r = req.prepare()
- >>> r
- <PreparedRequest [GET]>
-
- >>> s = requests.Session()
- >>> s.send(r)
- <Response [200]>
- """
-
- def __init__(self):
- #: HTTP verb to send to the server.
- self.method = None
- #: HTTP URL to send the request to.
- self.url = None
- #: dictionary of HTTP headers.
- self.headers = None
- # The `CookieJar` used to create the Cookie header will be stored here
- # after prepare_cookies is called
- self._cookies = None
- #: request body to send to the server.
- self.body = None
- #: dictionary of callback hooks, for internal usage.
- self.hooks = default_hooks()
- #: integer denoting starting position of a readable file-like body.
- self._body_position = None
-
- def prepare(
- self,
- method=None,
- url=None,
- headers=None,
- files=None,
- data=None,
- params=None,
- auth=None,
- cookies=None,
- hooks=None,
- json=None,
- ):
- """Prepares the entire request with the given parameters."""
-
- self.prepare_method(method)
- self.prepare_url(url, params)
- self.prepare_headers(headers)
- self.prepare_cookies(cookies)
- self.prepare_body(data, files, json)
- self.prepare_auth(auth, url)
-
- # Note that prepare_auth must be last to enable authentication schemes
- # such as OAuth to work on a fully prepared request.
-
- # This MUST go after prepare_auth. Authenticators could add a hook
- self.prepare_hooks(hooks)
-
- def __repr__(self):
- return f"<PreparedRequest [{self.method}]>"
-
- def copy(self):
- p = PreparedRequest()
- p.method = self.method
- p.url = self.url
- p.headers = self.headers.copy() if self.headers is not None else None
- p._cookies = _copy_cookie_jar(self._cookies)
- p.body = self.body
- p.hooks = self.hooks
- p._body_position = self._body_position
- return p
-
- def prepare_method(self, method):
- """Prepares the given HTTP method."""
- self.method = method
- if self.method is not None:
- self.method = to_native_string(self.method.upper())
-
- @staticmethod
- def _get_idna_encoded_host(host):
- import idna
-
- try:
- host = idna.encode(host, uts46=True).decode("utf-8")
- except idna.IDNAError:
- raise UnicodeError
- return host
-
- def prepare_url(self, url, params):
- """Prepares the given HTTP URL."""
- #: Accept objects that have string representations.
- #: We're unable to blindly call unicode/str functions
- #: as this will include the bytestring indicator (b'')
- #: on python 3.x.
- #: https://github.com/psf/requests/pull/2238
- if isinstance(url, bytes):
- url = url.decode("utf8")
- else:
- url = str(url)
-
- # Remove leading whitespaces from url
- url = url.lstrip()
-
- # Don't do any URL preparation for non-HTTP schemes like `mailto`,
- # `data` etc to work around exceptions from `url_parse`, which
- # handles RFC 3986 only.
- if ":" in url and not url.lower().startswith("http"):
- self.url = url
- return
-
- # Support for unicode domain names and paths.
- try:
- scheme, auth, host, port, path, query, fragment = parse_url(url)
- except LocationParseError as e:
- raise InvalidURL(*e.args)
-
- if not scheme:
- raise MissingSchema(
- f"Invalid URL {url!r}: No scheme supplied. "
- f"Perhaps you meant https://{url}?"
- )
-
- if not host:
- raise InvalidURL(f"Invalid URL {url!r}: No host supplied")
-
- # In general, we want to try IDNA encoding the hostname if the string contains
- # non-ASCII characters. This allows users to automatically get the correct IDNA
- # behaviour. For strings containing only ASCII characters, we need to also verify
- # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
- if not unicode_is_ascii(host):
- try:
- host = self._get_idna_encoded_host(host)
- except UnicodeError:
- raise InvalidURL("URL has an invalid label.")
- elif host.startswith(("*", ".")):
- raise InvalidURL("URL has an invalid label.")
-
- # Carefully reconstruct the network location
- netloc = auth or ""
- if netloc:
- netloc += "@"
- netloc += host
- if port:
- netloc += f":{port}"
-
- # Bare domains aren't valid URLs.
- if not path:
- path = "/"
-
- if isinstance(params, (str, bytes)):
- params = to_native_string(params)
-
- enc_params = self._encode_params(params)
- if enc_params:
- if query:
- query = f"{query}&{enc_params}"
- else:
- query = enc_params
-
- url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
- self.url = url
-
- def prepare_headers(self, headers):
- """Prepares the given HTTP headers."""
-
- self.headers = CaseInsensitiveDict()
- if headers:
- for header in headers.items():
- # Raise exception on invalid header value.
- check_header_validity(header)
- name, value = header
- self.headers[to_native_string(name)] = value
-
- def prepare_body(self, data, files, json=None):
- """Prepares the given HTTP body data."""
-
- # Check if file, fo, generator, iterator.
- # If not, run through normal process.
-
- # Nottin' on you.
- body = None
- content_type = None
-
- if not data and json is not None:
- # urllib3 requires a bytes-like body. Python 2's json.dumps
- # provides this natively, but Python 3 gives a Unicode string.
- content_type = "application/json"
-
- try:
- body = complexjson.dumps(json, allow_nan=False)
- except ValueError as ve:
- raise InvalidJSONError(ve, request=self)
-
- if not isinstance(body, bytes):
- body = body.encode("utf-8")
-
- is_stream = all(
- [
- hasattr(data, "__iter__"),
- not isinstance(data, (basestring, list, tuple, Mapping)),
- ]
- )
-
- if is_stream:
- try:
- length = super_len(data)
- except (TypeError, AttributeError, UnsupportedOperation):
- length = None
-
- body = data
-
- if getattr(body, "tell", None) is not None:
- # Record the current file position before reading.
- # This will allow us to rewind a file in the event
- # of a redirect.
- try:
- self._body_position = body.tell()
- except OSError:
- # This differentiates from None, allowing us to catch
- # a failed `tell()` later when trying to rewind the body
- self._body_position = object()
-
- if files:
- raise NotImplementedError(
- "Streamed bodies and files are mutually exclusive."
- )
-
- if length:
- self.headers["Content-Length"] = builtin_str(length)
- else:
- self.headers["Transfer-Encoding"] = "chunked"
- else:
- # Multi-part file uploads.
- if files:
- (body, content_type) = self._encode_files(files, data)
- else:
- if data:
- body = self._encode_params(data)
- if isinstance(data, basestring) or hasattr(data, "read"):
- content_type = None
- else:
- content_type = "application/x-www-form-urlencoded"
-
- self.prepare_content_length(body)
-
- # Add content-type if it wasn't explicitly provided.
- if content_type and ("content-type" not in self.headers):
- self.headers["Content-Type"] = content_type
-
- self.body = body
-
- def prepare_content_length(self, body):
- """Prepare Content-Length header based on request method and body"""
- if body is not None:
- length = super_len(body)
- if length:
- # If length exists, set it. Otherwise, we fallback
- # to Transfer-Encoding: chunked.
- self.headers["Content-Length"] = builtin_str(length)
- elif (
- self.method not in ("GET", "HEAD")
- and self.headers.get("Content-Length") is None
- ):
- # Set Content-Length to 0 for methods that can have a body
- # but don't provide one. (i.e. not GET or HEAD)
- self.headers["Content-Length"] = "0"
-
- def prepare_auth(self, auth, url=""):
- """Prepares the given HTTP auth data."""
-
- # If no Auth is explicitly provided, extract it from the URL first.
- if auth is None:
- url_auth = get_auth_from_url(self.url)
- auth = url_auth if any(url_auth) else None
-
- if auth:
- if isinstance(auth, tuple) and len(auth) == 2:
- # special-case basic HTTP auth
- auth = HTTPBasicAuth(*auth)
-
- # Allow auth to make its changes.
- r = auth(self)
-
- # Update self to reflect the auth changes.
- self.__dict__.update(r.__dict__)
-
- # Recompute Content-Length
- self.prepare_content_length(self.body)
-
- def prepare_cookies(self, cookies):
- """Prepares the given HTTP cookie data.
-
- This function eventually generates a ``Cookie`` header from the
- given cookies using cookielib. Due to cookielib's design, the header
- will not be regenerated if it already exists, meaning this function
- can only be called once for the life of the
- :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
- to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
- header is removed beforehand.
- """
- if isinstance(cookies, cookielib.CookieJar):
- self._cookies = cookies
- else:
- self._cookies = cookiejar_from_dict(cookies)
-
- cookie_header = get_cookie_header(self._cookies, self)
- if cookie_header is not None:
- self.headers["Cookie"] = cookie_header
-
- def prepare_hooks(self, hooks):
- """Prepares the given hooks."""
- # hooks can be passed as None to the prepare method and to this
- # method. To prevent iterating over None, simply use an empty list
- # if hooks is False-y
- hooks = hooks or []
- for event in hooks:
- self.register_hook(event, hooks[event])
-
-
- class Response:
- """The :class:`Response <Response>` object, which contains a
- server's response to an HTTP request.
- """
-
- __attrs__ = [
- "_content",
- "status_code",
- "headers",
- "url",
- "history",
- "encoding",
- "reason",
- "cookies",
- "elapsed",
- "request",
- ]
-
- def __init__(self):
- self._content = False
- self._content_consumed = False
- self._next = None
-
- #: Integer Code of responded HTTP Status, e.g. 404 or 200.
- self.status_code = None
-
- #: Case-insensitive Dictionary of Response Headers.
- #: For example, ``headers['content-encoding']`` will return the
- #: value of a ``'Content-Encoding'`` response header.
- self.headers = CaseInsensitiveDict()
-
- #: File-like object representation of response (for advanced usage).
- #: Use of ``raw`` requires that ``stream=True`` be set on the request.
- #: This requirement does not apply for use internally to Requests.
- self.raw = None
-
- #: Final URL location of Response.
- self.url = None
-
- #: Encoding to decode with when accessing r.text.
- self.encoding = None
-
- #: A list of :class:`Response <Response>` objects from
- #: the history of the Request. Any redirect responses will end
- #: up here. The list is sorted from the oldest to the most recent request.
- self.history = []
-
- #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
- self.reason = None
-
- #: A CookieJar of Cookies the server sent back.
- self.cookies = cookiejar_from_dict({})
-
- #: The amount of time elapsed between sending the request
- #: and the arrival of the response (as a timedelta).
- #: This property specifically measures the time taken between sending
- #: the first byte of the request and finishing parsing the headers. It
- #: is therefore unaffected by consuming the response content or the
- #: value of the ``stream`` keyword argument.
- self.elapsed = datetime.timedelta(0)
-
- #: The :class:`PreparedRequest <PreparedRequest>` object to which this
- #: is a response.
- self.request = None
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- self.close()
-
- def __getstate__(self):
- # Consume everything; accessing the content attribute makes
- # sure the content has been fully read.
- if not self._content_consumed:
- self.content
-
- return {attr: getattr(self, attr, None) for attr in self.__attrs__}
-
- def __setstate__(self, state):
- for name, value in state.items():
- setattr(self, name, value)
-
- # pickled objects do not have .raw
- setattr(self, "_content_consumed", True)
- setattr(self, "raw", None)
-
- def __repr__(self):
- return f"<Response [{self.status_code}]>"
-
- def __bool__(self):
- """Returns True if :attr:`status_code` is less than 400.
-
- This attribute checks if the status code of the response is between
- 400 and 600 to see if there was a client error or a server error. If
- the status code, is between 200 and 400, this will return True. This
- is **not** a check to see if the response code is ``200 OK``.
- """
- return self.ok
-
- def __nonzero__(self):
- """Returns True if :attr:`status_code` is less than 400.
-
- This attribute checks if the status code of the response is between
- 400 and 600 to see if there was a client error or a server error. If
- the status code, is between 200 and 400, this will return True. This
- is **not** a check to see if the response code is ``200 OK``.
- """
- return self.ok
-
- def __iter__(self):
- """Allows you to use a response as an iterator."""
- return self.iter_content(128)
-
- @property
- def ok(self):
- """Returns True if :attr:`status_code` is less than 400, False if not.
-
- This attribute checks if the status code of the response is between
- 400 and 600 to see if there was a client error or a server error. If
- the status code is between 200 and 400, this will return True. This
- is **not** a check to see if the response code is ``200 OK``.
- """
- try:
- self.raise_for_status()
- except HTTPError:
- return False
- return True
-
- @property
- def is_redirect(self):
- """True if this Response is a well-formed HTTP redirect that could have
- been processed automatically (by :meth:`Session.resolve_redirects`).
- """
- return "location" in self.headers and self.status_code in REDIRECT_STATI
-
- @property
- def is_permanent_redirect(self):
- """True if this Response one of the permanent versions of redirect."""
- return "location" in self.headers and self.status_code in (
- codes.moved_permanently,
- codes.permanent_redirect,
- )
-
- @property
- def next(self):
- """Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
- return self._next
-
- @property
- def apparent_encoding(self):
- """The apparent encoding, provided by the charset_normalizer or chardet libraries."""
- return chardet.detect(self.content)["encoding"]
-
- def iter_content(self, chunk_size=1, decode_unicode=False):
- """Iterates over the response data. When stream=True is set on the
- request, this avoids reading the content at once into memory for
- large responses. The chunk size is the number of bytes it should
- read into memory. This is not necessarily the length of each item
- returned as decoding can take place.
-
- chunk_size must be of type int or None. A value of None will
- function differently depending on the value of `stream`.
- stream=True will read data as it arrives in whatever size the
- chunks are received. If stream=False, data is returned as
- a single chunk.
-
- If decode_unicode is True, content will be decoded using the best
- available encoding based on the response.
- """
-
- def generate():
- # Special case for urllib3.
- if hasattr(self.raw, "stream"):
- try:
- yield from self.raw.stream(chunk_size, decode_content=True)
- except ProtocolError as e:
- raise ChunkedEncodingError(e)
- except DecodeError as e:
- raise ContentDecodingError(e)
- except ReadTimeoutError as e:
- raise ConnectionError(e)
- except SSLError as e:
- raise RequestsSSLError(e)
- else:
- # Standard file-like object.
- while True:
- chunk = self.raw.read(chunk_size)
- if not chunk:
- break
- yield chunk
-
- self._content_consumed = True
-
- if self._content_consumed and isinstance(self._content, bool):
- raise StreamConsumedError()
- elif chunk_size is not None and not isinstance(chunk_size, int):
- raise TypeError(
- f"chunk_size must be an int, it is instead a {type(chunk_size)}."
- )
- # simulate reading small chunks of the content
- reused_chunks = iter_slices(self._content, chunk_size)
-
- stream_chunks = generate()
-
- chunks = reused_chunks if self._content_consumed else stream_chunks
-
- if decode_unicode:
- chunks = stream_decode_response_unicode(chunks, self)
-
- return chunks
-
- def iter_lines(
- self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None
- ):
- """Iterates over the response data, one line at a time. When
- stream=True is set on the request, this avoids reading the
- content at once into memory for large responses.
-
- .. note:: This method is not reentrant safe.
- """
-
- pending = None
-
- for chunk in self.iter_content(
- chunk_size=chunk_size, decode_unicode=decode_unicode
- ):
-
- if pending is not None:
- chunk = pending + chunk
-
- if delimiter:
- lines = chunk.split(delimiter)
- else:
- lines = chunk.splitlines()
-
- if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
- pending = lines.pop()
- else:
- pending = None
-
- yield from lines
-
- if pending is not None:
- yield pending
-
- @property
- def content(self):
- """Content of the response, in bytes."""
-
- if self._content is False:
- # Read the contents.
- if self._content_consumed:
- raise RuntimeError("The content for this response was already consumed")
-
- if self.status_code == 0 or self.raw is None:
- self._content = None
- else:
- self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
-
- self._content_consumed = True
- # don't need to release the connection; that's been handled by urllib3
- # since we exhausted the data.
- return self._content
-
- @property
- def text(self):
- """Content of the response, in unicode.
-
- If Response.encoding is None, encoding will be guessed using
- ``charset_normalizer`` or ``chardet``.
-
- The encoding of the response content is determined based solely on HTTP
- headers, following RFC 2616 to the letter. If you can take advantage of
- non-HTTP knowledge to make a better guess at the encoding, you should
- set ``r.encoding`` appropriately before accessing this property.
- """
-
- # Try charset from content-type
- content = None
- encoding = self.encoding
-
- if not self.content:
- return ""
-
- # Fallback to auto-detected encoding.
- if self.encoding is None:
- encoding = self.apparent_encoding
-
- # Decode unicode from given encoding.
- try:
- content = str(self.content, encoding, errors="replace")
- except (LookupError, TypeError):
- # A LookupError is raised if the encoding was not found which could
- # indicate a misspelling or similar mistake.
- #
- # A TypeError can be raised if encoding is None
- #
- # So we try blindly encoding.
- content = str(self.content, errors="replace")
-
- return content
-
- def json(self, **kwargs):
- r"""Returns the json-encoded content of a response, if any.
-
- :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
- :raises requests.exceptions.JSONDecodeError: If the response body does not
- contain valid json.
- """
-
- if not self.encoding and self.content and len(self.content) > 3:
- # No encoding set. JSON RFC 4627 section 3 states we should expect
- # UTF-8, -16 or -32. Detect which one to use; If the detection or
- # decoding fails, fall back to `self.text` (using charset_normalizer to make
- # a best guess).
- encoding = guess_json_utf(self.content)
- if encoding is not None:
- try:
- return complexjson.loads(self.content.decode(encoding), **kwargs)
- except UnicodeDecodeError:
- # Wrong UTF codec detected; usually because it's not UTF-8
- # but some other 8-bit codec. This is an RFC violation,
- # and the server didn't bother to tell us what codec *was*
- # used.
- pass
- except JSONDecodeError as e:
- raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
-
- try:
- return complexjson.loads(self.text, **kwargs)
- except JSONDecodeError as e:
- # Catch JSON-related errors and raise as requests.JSONDecodeError
- # This aliases json.JSONDecodeError and simplejson.JSONDecodeError
- raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
-
- @property
- def links(self):
- """Returns the parsed header links of the response, if any."""
-
- header = self.headers.get("link")
-
- resolved_links = {}
-
- if header:
- links = parse_header_links(header)
-
- for link in links:
- key = link.get("rel") or link.get("url")
- resolved_links[key] = link
-
- return resolved_links
-
- def raise_for_status(self):
- """Raises :class:`HTTPError`, if one occurred."""
-
- http_error_msg = ""
- if isinstance(self.reason, bytes):
- # We attempt to decode utf-8 first because some servers
- # choose to localize their reason strings. If the string
- # isn't utf-8, we fall back to iso-8859-1 for all other
- # encodings. (See PR #3538)
- try:
- reason = self.reason.decode("utf-8")
- except UnicodeDecodeError:
- reason = self.reason.decode("iso-8859-1")
- else:
- reason = self.reason
-
- if 400 <= self.status_code < 500:
- http_error_msg = (
- f"{self.status_code} Client Error: {reason} for url: {self.url}"
- )
-
- elif 500 <= self.status_code < 600:
- http_error_msg = (
- f"{self.status_code} Server Error: {reason} for url: {self.url}"
- )
-
- if http_error_msg:
- raise HTTPError(http_error_msg, response=self)
-
- def close(self):
- """Releases the connection back to the pool. Once this method has been
- called the underlying ``raw`` object must not be accessed again.
-
- *Note: Should not normally need to be called explicitly.*
- """
- if not self._content_consumed:
- self.raw.close()
-
- release_conn = getattr(self.raw, "release_conn", None)
- if release_conn is not None:
- release_conn()
|