Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

five.py 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. # -*- coding: utf-8 -*-
  2. """
  3. celery.five
  4. ~~~~~~~~~~~
  5. Compatibility implementations of features
  6. only available in newer Python versions.
  7. """
  8. from __future__ import absolute_import
  9. import io
  10. import operator
  11. import sys
  12. from importlib import import_module
  13. from types import ModuleType
  14. from kombu.five import monotonic
  15. try:
  16. from collections import Counter
  17. except ImportError: # pragma: no cover
  18. from collections import defaultdict
  19. def Counter(): # noqa
  20. return defaultdict(int)
  21. __all__ = ['Counter', 'reload', 'UserList', 'UserDict', 'Queue', 'Empty',
  22. 'zip_longest', 'map', 'string', 'string_t',
  23. 'long_t', 'text_t', 'range', 'int_types', 'items', 'keys', 'values',
  24. 'nextfun', 'reraise', 'WhateverIO', 'with_metaclass',
  25. 'OrderedDict', 'THREAD_TIMEOUT_MAX', 'format_d',
  26. 'class_property', 'reclassmethod', 'create_module',
  27. 'recreate_module', 'monotonic']
  28. # ############# py3k #########################################################
  29. PY3 = sys.version_info[0] == 3
  30. try:
  31. reload = reload # noqa
  32. except NameError: # pragma: no cover
  33. from imp import reload # noqa
  34. try:
  35. from UserList import UserList # noqa
  36. except ImportError: # pragma: no cover
  37. from collections import UserList # noqa
  38. try:
  39. from UserDict import UserDict # noqa
  40. except ImportError: # pragma: no cover
  41. from collections import UserDict # noqa
  42. if PY3: # pragma: no cover
  43. import builtins
  44. from queue import Queue, Empty
  45. from itertools import zip_longest
  46. map = map
  47. string = str
  48. string_t = str
  49. long_t = int
  50. text_t = str
  51. range = range
  52. int_types = (int, )
  53. _byte_t = bytes
  54. open_fqdn = 'builtins.open'
  55. def items(d):
  56. return d.items()
  57. def keys(d):
  58. return d.keys()
  59. def values(d):
  60. return d.values()
  61. def nextfun(it):
  62. return it.__next__
  63. exec_ = getattr(builtins, 'exec')
  64. def reraise(tp, value, tb=None):
  65. if value.__traceback__ is not tb:
  66. raise value.with_traceback(tb)
  67. raise value
  68. else:
  69. import __builtin__ as builtins # noqa
  70. from Queue import Queue, Empty # noqa
  71. from itertools import imap as map, izip_longest as zip_longest # noqa
  72. string = unicode # noqa
  73. string_t = basestring # noqa
  74. text_t = unicode # noqa
  75. long_t = long # noqa
  76. range = xrange # noqa
  77. int_types = (int, long) # noqa
  78. _byte_t = (str, bytes) # noqa
  79. open_fqdn = '__builtin__.open'
  80. def items(d): # noqa
  81. return d.iteritems()
  82. def keys(d): # noqa
  83. return d.iterkeys()
  84. def values(d): # noqa
  85. return d.itervalues()
  86. def nextfun(it): # noqa
  87. return it.next
  88. def exec_(code, globs=None, locs=None): # pragma: no cover
  89. """Execute code in a namespace."""
  90. if globs is None:
  91. frame = sys._getframe(1)
  92. globs = frame.f_globals
  93. if locs is None:
  94. locs = frame.f_locals
  95. del frame
  96. elif locs is None:
  97. locs = globs
  98. exec("""exec code in globs, locs""")
  99. exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")
  100. def with_metaclass(Type, skip_attrs=set(['__dict__', '__weakref__'])):
  101. """Class decorator to set metaclass.
  102. Works with both Python 2 and Python 3 and it does not add
  103. an extra class in the lookup order like ``six.with_metaclass`` does
  104. (that is -- it copies the original class instead of using inheritance).
  105. """
  106. def _clone_with_metaclass(Class):
  107. attrs = dict((key, value) for key, value in items(vars(Class))
  108. if key not in skip_attrs)
  109. return Type(Class.__name__, Class.__bases__, attrs)
  110. return _clone_with_metaclass
  111. # ############# collections.OrderedDict ######################################
  112. # was moved to kombu
  113. from kombu.utils.compat import OrderedDict # noqa
  114. # ############# threading.TIMEOUT_MAX ########################################
  115. try:
  116. from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
  117. except ImportError:
  118. THREAD_TIMEOUT_MAX = 1e10 # noqa
  119. # ############# format(int, ',d') ############################################
  120. if sys.version_info >= (2, 7): # pragma: no cover
  121. def format_d(i):
  122. return format(i, ',d')
  123. else: # pragma: no cover
  124. def format_d(i): # noqa
  125. s = '%d' % i
  126. groups = []
  127. while s and s[-1].isdigit():
  128. groups.append(s[-3:])
  129. s = s[:-3]
  130. return s + ','.join(reversed(groups))
  131. # ############# Module Generation ############################################
  132. # Utilities to dynamically
  133. # recreate modules, either for lazy loading or
  134. # to create old modules at runtime instead of
  135. # having them litter the source tree.
  136. # import fails in python 2.5. fallback to reduce in stdlib
  137. try:
  138. from functools import reduce
  139. except ImportError:
  140. pass
  141. MODULE_DEPRECATED = """
  142. The module %s is deprecated and will be removed in a future version.
  143. """
  144. DEFAULT_ATTRS = set(['__file__', '__path__', '__doc__', '__all__'])
  145. # im_func is no longer available in Py3.
  146. # instead the unbound method itself can be used.
  147. if sys.version_info[0] == 3: # pragma: no cover
  148. def fun_of_method(method):
  149. return method
  150. else:
  151. def fun_of_method(method): # noqa
  152. return method.im_func
  153. def getappattr(path):
  154. """Gets attribute from the current_app recursively,
  155. e.g. getappattr('amqp.get_task_consumer')``."""
  156. from celery import current_app
  157. return current_app._rgetattr(path)
  158. def _compat_task_decorator(*args, **kwargs):
  159. from celery import current_app
  160. kwargs.setdefault('accept_magic_kwargs', True)
  161. return current_app.task(*args, **kwargs)
  162. def _compat_periodic_task_decorator(*args, **kwargs):
  163. from celery.task import periodic_task
  164. kwargs.setdefault('accept_magic_kwargs', True)
  165. return periodic_task(*args, **kwargs)
  166. COMPAT_MODULES = {
  167. 'celery': {
  168. 'execute': {
  169. 'send_task': 'send_task',
  170. },
  171. 'decorators': {
  172. 'task': _compat_task_decorator,
  173. 'periodic_task': _compat_periodic_task_decorator,
  174. },
  175. 'log': {
  176. 'get_default_logger': 'log.get_default_logger',
  177. 'setup_logger': 'log.setup_logger',
  178. 'setup_logging_subsystem': 'log.setup_logging_subsystem',
  179. 'redirect_stdouts_to_logger': 'log.redirect_stdouts_to_logger',
  180. },
  181. 'messaging': {
  182. 'TaskPublisher': 'amqp.TaskPublisher',
  183. 'TaskConsumer': 'amqp.TaskConsumer',
  184. 'establish_connection': 'connection',
  185. 'get_consumer_set': 'amqp.TaskConsumer',
  186. },
  187. 'registry': {
  188. 'tasks': 'tasks',
  189. },
  190. },
  191. 'celery.task': {
  192. 'control': {
  193. 'broadcast': 'control.broadcast',
  194. 'rate_limit': 'control.rate_limit',
  195. 'time_limit': 'control.time_limit',
  196. 'ping': 'control.ping',
  197. 'revoke': 'control.revoke',
  198. 'discard_all': 'control.purge',
  199. 'inspect': 'control.inspect',
  200. },
  201. 'schedules': 'celery.schedules',
  202. 'chords': 'celery.canvas',
  203. }
  204. }
  205. class class_property(object):
  206. def __init__(self, getter=None, setter=None):
  207. if getter is not None and not isinstance(getter, classmethod):
  208. getter = classmethod(getter)
  209. if setter is not None and not isinstance(setter, classmethod):
  210. setter = classmethod(setter)
  211. self.__get = getter
  212. self.__set = setter
  213. info = getter.__get__(object) # just need the info attrs.
  214. self.__doc__ = info.__doc__
  215. self.__name__ = info.__name__
  216. self.__module__ = info.__module__
  217. def __get__(self, obj, type=None):
  218. if obj and type is None:
  219. type = obj.__class__
  220. return self.__get.__get__(obj, type)()
  221. def __set__(self, obj, value):
  222. if obj is None:
  223. return self
  224. return self.__set.__get__(obj)(value)
  225. def setter(self, setter):
  226. return self.__class__(self.__get, setter)
  227. def reclassmethod(method):
  228. return classmethod(fun_of_method(method))
  229. class LazyModule(ModuleType):
  230. _compat_modules = ()
  231. _all_by_module = {}
  232. _direct = {}
  233. _object_origins = {}
  234. def __getattr__(self, name):
  235. if name in self._object_origins:
  236. module = __import__(self._object_origins[name], None, None, [name])
  237. for item in self._all_by_module[module.__name__]:
  238. setattr(self, item, getattr(module, item))
  239. return getattr(module, name)
  240. elif name in self._direct: # pragma: no cover
  241. module = __import__(self._direct[name], None, None, [name])
  242. setattr(self, name, module)
  243. return module
  244. return ModuleType.__getattribute__(self, name)
  245. def __dir__(self):
  246. return list(set(self.__all__) | DEFAULT_ATTRS)
  247. def __reduce__(self):
  248. return import_module, (self.__name__, )
  249. def create_module(name, attrs, cls_attrs=None, pkg=None,
  250. base=LazyModule, prepare_attr=None):
  251. fqdn = '.'.join([pkg.__name__, name]) if pkg else name
  252. cls_attrs = {} if cls_attrs is None else cls_attrs
  253. pkg, _, modname = name.rpartition('.')
  254. cls_attrs['__module__'] = pkg
  255. attrs = dict((attr_name, prepare_attr(attr) if prepare_attr else attr)
  256. for attr_name, attr in items(attrs))
  257. module = sys.modules[fqdn] = type(modname, (base, ), cls_attrs)(fqdn)
  258. module.__dict__.update(attrs)
  259. return module
  260. def recreate_module(name, compat_modules=(), by_module={}, direct={},
  261. base=LazyModule, **attrs):
  262. old_module = sys.modules[name]
  263. origins = get_origins(by_module)
  264. compat_modules = COMPAT_MODULES.get(name, ())
  265. cattrs = dict(
  266. _compat_modules=compat_modules,
  267. _all_by_module=by_module, _direct=direct,
  268. _object_origins=origins,
  269. __all__=tuple(set(reduce(
  270. operator.add,
  271. [tuple(v) for v in [compat_modules, origins, direct, attrs]],
  272. ))),
  273. )
  274. new_module = create_module(name, attrs, cls_attrs=cattrs, base=base)
  275. new_module.__dict__.update(dict((mod, get_compat_module(new_module, mod))
  276. for mod in compat_modules))
  277. return old_module, new_module
  278. def get_compat_module(pkg, name):
  279. from .local import Proxy
  280. def prepare(attr):
  281. if isinstance(attr, string_t):
  282. return Proxy(getappattr, (attr, ))
  283. return attr
  284. attrs = COMPAT_MODULES[pkg.__name__][name]
  285. if isinstance(attrs, string_t):
  286. fqdn = '.'.join([pkg.__name__, name])
  287. module = sys.modules[fqdn] = import_module(attrs)
  288. return module
  289. attrs['__all__'] = list(attrs)
  290. return create_module(name, dict(attrs), pkg=pkg, prepare_attr=prepare)
  291. def get_origins(defs):
  292. origins = {}
  293. for module, attrs in items(defs):
  294. origins.update(dict((attr, module) for attr in attrs))
  295. return origins
  296. _SIO_write = io.StringIO.write
  297. _SIO_init = io.StringIO.__init__
  298. class WhateverIO(io.StringIO):
  299. def __init__(self, v=None, *a, **kw):
  300. _SIO_init(self, v.decode() if isinstance(v, _byte_t) else v, *a, **kw)
  301. def write(self, data):
  302. _SIO_write(self, data.decode() if isinstance(data, _byte_t) else data)