You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

_collections.py 10.0KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. from __future__ import absolute_import
  2. from collections import Mapping, MutableMapping
  3. try:
  4. from threading import RLock
  5. except ImportError: # Platform-specific: No threads available
  6. class RLock:
  7. def __enter__(self):
  8. pass
  9. def __exit__(self, exc_type, exc_value, traceback):
  10. pass
  11. try: # Python 2.7+
  12. from collections import OrderedDict
  13. except ImportError:
  14. from .packages.ordered_dict import OrderedDict
  15. from .packages.six import iterkeys, itervalues, PY3
  16. __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
  17. _Null = object()
  18. class RecentlyUsedContainer(MutableMapping):
  19. """
  20. Provides a thread-safe dict-like container which maintains up to
  21. ``maxsize`` keys while throwing away the least-recently-used keys beyond
  22. ``maxsize``.
  23. :param maxsize:
  24. Maximum number of recent elements to retain.
  25. :param dispose_func:
  26. Every time an item is evicted from the container,
  27. ``dispose_func(value)`` is called. Callback which will get called
  28. """
  29. ContainerCls = OrderedDict
  30. def __init__(self, maxsize=10, dispose_func=None):
  31. self._maxsize = maxsize
  32. self.dispose_func = dispose_func
  33. self._container = self.ContainerCls()
  34. self.lock = RLock()
  35. def __getitem__(self, key):
  36. # Re-insert the item, moving it to the end of the eviction line.
  37. with self.lock:
  38. item = self._container.pop(key)
  39. self._container[key] = item
  40. return item
  41. def __setitem__(self, key, value):
  42. evicted_value = _Null
  43. with self.lock:
  44. # Possibly evict the existing value of 'key'
  45. evicted_value = self._container.get(key, _Null)
  46. self._container[key] = value
  47. # If we didn't evict an existing value, we might have to evict the
  48. # least recently used item from the beginning of the container.
  49. if len(self._container) > self._maxsize:
  50. _key, evicted_value = self._container.popitem(last=False)
  51. if self.dispose_func and evicted_value is not _Null:
  52. self.dispose_func(evicted_value)
  53. def __delitem__(self, key):
  54. with self.lock:
  55. value = self._container.pop(key)
  56. if self.dispose_func:
  57. self.dispose_func(value)
  58. def __len__(self):
  59. with self.lock:
  60. return len(self._container)
  61. def __iter__(self):
  62. raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
  63. def clear(self):
  64. with self.lock:
  65. # Copy pointers to all values, then wipe the mapping
  66. values = list(itervalues(self._container))
  67. self._container.clear()
  68. if self.dispose_func:
  69. for value in values:
  70. self.dispose_func(value)
  71. def keys(self):
  72. with self.lock:
  73. return list(iterkeys(self._container))
  74. class HTTPHeaderDict(MutableMapping):
  75. """
  76. :param headers:
  77. An iterable of field-value pairs. Must not contain multiple field names
  78. when compared case-insensitively.
  79. :param kwargs:
  80. Additional field-value pairs to pass in to ``dict.update``.
  81. A ``dict`` like container for storing HTTP Headers.
  82. Field names are stored and compared case-insensitively in compliance with
  83. RFC 7230. Iteration provides the first case-sensitive key seen for each
  84. case-insensitive pair.
  85. Using ``__setitem__`` syntax overwrites fields that compare equal
  86. case-insensitively in order to maintain ``dict``'s api. For fields that
  87. compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
  88. in a loop.
  89. If multiple fields that are equal case-insensitively are passed to the
  90. constructor or ``.update``, the behavior is undefined and some will be
  91. lost.
  92. >>> headers = HTTPHeaderDict()
  93. >>> headers.add('Set-Cookie', 'foo=bar')
  94. >>> headers.add('set-cookie', 'baz=quxx')
  95. >>> headers['content-length'] = '7'
  96. >>> headers['SET-cookie']
  97. 'foo=bar, baz=quxx'
  98. >>> headers['Content-Length']
  99. '7'
  100. """
  101. def __init__(self, headers=None, **kwargs):
  102. super(HTTPHeaderDict, self).__init__()
  103. self._container = OrderedDict()
  104. if headers is not None:
  105. if isinstance(headers, HTTPHeaderDict):
  106. self._copy_from(headers)
  107. else:
  108. self.extend(headers)
  109. if kwargs:
  110. self.extend(kwargs)
  111. def __setitem__(self, key, val):
  112. self._container[key.lower()] = [key, val]
  113. return self._container[key.lower()]
  114. def __getitem__(self, key):
  115. val = self._container[key.lower()]
  116. return ', '.join(val[1:])
  117. def __delitem__(self, key):
  118. del self._container[key.lower()]
  119. def __contains__(self, key):
  120. return key.lower() in self._container
  121. def __eq__(self, other):
  122. if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
  123. return False
  124. if not isinstance(other, type(self)):
  125. other = type(self)(other)
  126. return (dict((k.lower(), v) for k, v in self.itermerged()) ==
  127. dict((k.lower(), v) for k, v in other.itermerged()))
  128. def __ne__(self, other):
  129. return not self.__eq__(other)
  130. if not PY3: # Python 2
  131. iterkeys = MutableMapping.iterkeys
  132. itervalues = MutableMapping.itervalues
  133. __marker = object()
  134. def __len__(self):
  135. return len(self._container)
  136. def __iter__(self):
  137. # Only provide the originally cased names
  138. for vals in self._container.values():
  139. yield vals[0]
  140. def pop(self, key, default=__marker):
  141. '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
  142. If key is not found, d is returned if given, otherwise KeyError is raised.
  143. '''
  144. # Using the MutableMapping function directly fails due to the private marker.
  145. # Using ordinary dict.pop would expose the internal structures.
  146. # So let's reinvent the wheel.
  147. try:
  148. value = self[key]
  149. except KeyError:
  150. if default is self.__marker:
  151. raise
  152. return default
  153. else:
  154. del self[key]
  155. return value
  156. def discard(self, key):
  157. try:
  158. del self[key]
  159. except KeyError:
  160. pass
  161. def add(self, key, val):
  162. """Adds a (name, value) pair, doesn't overwrite the value if it already
  163. exists.
  164. >>> headers = HTTPHeaderDict(foo='bar')
  165. >>> headers.add('Foo', 'baz')
  166. >>> headers['foo']
  167. 'bar, baz'
  168. """
  169. key_lower = key.lower()
  170. new_vals = [key, val]
  171. # Keep the common case aka no item present as fast as possible
  172. vals = self._container.setdefault(key_lower, new_vals)
  173. if new_vals is not vals:
  174. vals.append(val)
  175. def extend(self, *args, **kwargs):
  176. """Generic import function for any type of header-like object.
  177. Adapted version of MutableMapping.update in order to insert items
  178. with self.add instead of self.__setitem__
  179. """
  180. if len(args) > 1:
  181. raise TypeError("extend() takes at most 1 positional "
  182. "arguments ({0} given)".format(len(args)))
  183. other = args[0] if len(args) >= 1 else ()
  184. if isinstance(other, HTTPHeaderDict):
  185. for key, val in other.iteritems():
  186. self.add(key, val)
  187. elif isinstance(other, Mapping):
  188. for key in other:
  189. self.add(key, other[key])
  190. elif hasattr(other, "keys"):
  191. for key in other.keys():
  192. self.add(key, other[key])
  193. else:
  194. for key, value in other:
  195. self.add(key, value)
  196. for key, value in kwargs.items():
  197. self.add(key, value)
  198. def getlist(self, key, default=__marker):
  199. """Returns a list of all the values for the named field. Returns an
  200. empty list if the key doesn't exist."""
  201. try:
  202. vals = self._container[key.lower()]
  203. except KeyError:
  204. if default is self.__marker:
  205. return []
  206. return default
  207. else:
  208. return vals[1:]
  209. # Backwards compatibility for httplib
  210. getheaders = getlist
  211. getallmatchingheaders = getlist
  212. iget = getlist
  213. # Backwards compatibility for http.cookiejar
  214. get_all = getlist
  215. def __repr__(self):
  216. return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
  217. def _copy_from(self, other):
  218. for key in other:
  219. val = other.getlist(key)
  220. if isinstance(val, list):
  221. # Don't need to convert tuples
  222. val = list(val)
  223. self._container[key.lower()] = [key] + val
  224. def copy(self):
  225. clone = type(self)()
  226. clone._copy_from(self)
  227. return clone
  228. def iteritems(self):
  229. """Iterate over all header lines, including duplicate ones."""
  230. for key in self:
  231. vals = self._container[key.lower()]
  232. for val in vals[1:]:
  233. yield vals[0], val
  234. def itermerged(self):
  235. """Iterate over all headers, merging duplicate ones together."""
  236. for key in self:
  237. val = self._container[key.lower()]
  238. yield val[0], ', '.join(val[1:])
  239. def items(self):
  240. return list(self.iteritems())
  241. @classmethod
  242. def from_httplib(cls, message): # Python 2
  243. """Read headers from a Python 2 httplib message object."""
  244. # python2.7 does not expose a proper API for exporting multiheaders
  245. # efficiently. This function re-reads raw lines from the message
  246. # object and extracts the multiheaders properly.
  247. headers = []
  248. for line in message.headers:
  249. if line.startswith((' ', '\t')):
  250. key, value = headers[-1]
  251. headers[-1] = (key, value + '\r\n' + line.rstrip())
  252. continue
  253. key, value = line.split(':', 1)
  254. headers.append((key, value.strip()))
  255. return cls(headers)