Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

process.py 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. #
  2. # Module providing the `Process` class which emulates `threading.Thread`
  3. #
  4. # multiprocessing/process.py
  5. #
  6. # Copyright (c) 2006-2008, R Oudkerk
  7. # Licensed to PSF under a Contributor Agreement.
  8. #
  9. from __future__ import absolute_import
  10. #
  11. # Imports
  12. #
  13. import os
  14. import sys
  15. import signal
  16. import itertools
  17. import binascii
  18. import logging
  19. import threading
  20. from multiprocessing import process as _mproc
  21. from .compat import bytes
  22. try:
  23. from _weakrefset import WeakSet
  24. except ImportError:
  25. WeakSet = None # noqa
  26. from .five import items, string_t
  27. try:
  28. ORIGINAL_DIR = os.path.abspath(os.getcwd())
  29. except OSError:
  30. ORIGINAL_DIR = None
  31. __all__ = ['Process', 'current_process', 'active_children']
  32. #
  33. # Public functions
  34. #
  35. def current_process():
  36. '''
  37. Return process object representing the current process
  38. '''
  39. return _current_process
  40. def _set_current_process(process):
  41. global _current_process
  42. _current_process = _mproc._current_process = process
  43. def _cleanup():
  44. # check for processes which have finished
  45. if _current_process is not None:
  46. for p in list(_current_process._children):
  47. if p._popen.poll() is not None:
  48. _current_process._children.discard(p)
  49. def _maybe_flush(f):
  50. try:
  51. f.flush()
  52. except (AttributeError, EnvironmentError, NotImplementedError):
  53. pass
  54. def active_children(_cleanup=_cleanup):
  55. '''
  56. Return list of process objects corresponding to live child processes
  57. '''
  58. try:
  59. _cleanup()
  60. except TypeError:
  61. # called after gc collect so _cleanup does not exist anymore
  62. return []
  63. if _current_process is not None:
  64. return list(_current_process._children)
  65. return []
  66. class Process(object):
  67. '''
  68. Process objects represent activity that is run in a separate process
  69. The class is analagous to `threading.Thread`
  70. '''
  71. _Popen = None
  72. def __init__(self, group=None, target=None, name=None,
  73. args=(), kwargs={}, daemon=None, **_kw):
  74. assert group is None, 'group argument must be None for now'
  75. count = next(_current_process._counter)
  76. self._identity = _current_process._identity + (count,)
  77. self._authkey = _current_process._authkey
  78. if daemon is not None:
  79. self._daemonic = daemon
  80. else:
  81. self._daemonic = _current_process._daemonic
  82. self._tempdir = _current_process._tempdir
  83. self._semprefix = _current_process._semprefix
  84. self._unlinkfd = _current_process._unlinkfd
  85. self._parent_pid = os.getpid()
  86. self._popen = None
  87. self._target = target
  88. self._args = tuple(args)
  89. self._kwargs = dict(kwargs)
  90. self._name = (
  91. name or type(self).__name__ + '-' +
  92. ':'.join(str(i) for i in self._identity)
  93. )
  94. if _dangling is not None:
  95. _dangling.add(self)
  96. def run(self):
  97. '''
  98. Method to be run in sub-process; can be overridden in sub-class
  99. '''
  100. if self._target:
  101. self._target(*self._args, **self._kwargs)
  102. def start(self):
  103. '''
  104. Start child process
  105. '''
  106. assert self._popen is None, 'cannot start a process twice'
  107. assert self._parent_pid == os.getpid(), \
  108. 'can only start a process object created by current process'
  109. _cleanup()
  110. if self._Popen is not None:
  111. Popen = self._Popen
  112. else:
  113. from .forking import Popen
  114. self._popen = Popen(self)
  115. self._sentinel = self._popen.sentinel
  116. _current_process._children.add(self)
  117. def terminate(self):
  118. '''
  119. Terminate process; sends SIGTERM signal or uses TerminateProcess()
  120. '''
  121. self._popen.terminate()
  122. def join(self, timeout=None):
  123. '''
  124. Wait until child process terminates
  125. '''
  126. assert self._parent_pid == os.getpid(), 'can only join a child process'
  127. assert self._popen is not None, 'can only join a started process'
  128. res = self._popen.wait(timeout)
  129. if res is not None:
  130. _current_process._children.discard(self)
  131. def is_alive(self):
  132. '''
  133. Return whether process is alive
  134. '''
  135. if self is _current_process:
  136. return True
  137. assert self._parent_pid == os.getpid(), 'can only test a child process'
  138. if self._popen is None:
  139. return False
  140. self._popen.poll()
  141. return self._popen.returncode is None
  142. def _is_alive(self):
  143. if self._popen is None:
  144. return False
  145. return self._popen.poll() is None
  146. def _get_name(self):
  147. return self._name
  148. def _set_name(self, value):
  149. assert isinstance(name, string_t), 'name must be a string'
  150. self._name = value
  151. name = property(_get_name, _set_name)
  152. def _get_daemon(self):
  153. return self._daemonic
  154. def _set_daemon(self, daemonic):
  155. assert self._popen is None, 'process has already started'
  156. self._daemonic = daemonic
  157. daemon = property(_get_daemon, _set_daemon)
  158. def _get_authkey(self):
  159. return self._authkey
  160. def _set_authkey(self, authkey):
  161. self._authkey = AuthenticationString(authkey)
  162. authkey = property(_get_authkey, _set_authkey)
  163. @property
  164. def exitcode(self):
  165. '''
  166. Return exit code of process or `None` if it has yet to stop
  167. '''
  168. if self._popen is None:
  169. return self._popen
  170. return self._popen.poll()
  171. @property
  172. def ident(self):
  173. '''
  174. Return identifier (PID) of process or `None` if it has yet to start
  175. '''
  176. if self is _current_process:
  177. return os.getpid()
  178. else:
  179. return self._popen and self._popen.pid
  180. pid = ident
  181. @property
  182. def sentinel(self):
  183. '''
  184. Return a file descriptor (Unix) or handle (Windows) suitable for
  185. waiting for process termination.
  186. '''
  187. try:
  188. return self._sentinel
  189. except AttributeError:
  190. raise ValueError("process not started")
  191. def __repr__(self):
  192. if self is _current_process:
  193. status = 'started'
  194. elif self._parent_pid != os.getpid():
  195. status = 'unknown'
  196. elif self._popen is None:
  197. status = 'initial'
  198. else:
  199. if self._popen.poll() is not None:
  200. status = self.exitcode
  201. else:
  202. status = 'started'
  203. if type(status) is int:
  204. if status == 0:
  205. status = 'stopped'
  206. else:
  207. status = 'stopped[%s]' % _exitcode_to_name.get(status, status)
  208. return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,
  209. status, self._daemonic and ' daemon' or '')
  210. ##
  211. def _bootstrap(self):
  212. from . import util
  213. global _current_process
  214. try:
  215. self._children = set()
  216. self._counter = itertools.count(1)
  217. if sys.stdin is not None:
  218. try:
  219. sys.stdin.close()
  220. sys.stdin = open(os.devnull)
  221. except (OSError, ValueError):
  222. pass
  223. old_process = _current_process
  224. _set_current_process(self)
  225. # Re-init logging system.
  226. # Workaround for http://bugs.python.org/issue6721/#msg140215
  227. # Python logging module uses RLock() objects which are broken
  228. # after fork. This can result in a deadlock (Celery Issue #496).
  229. loggerDict = logging.Logger.manager.loggerDict
  230. logger_names = list(loggerDict.keys())
  231. logger_names.append(None) # for root logger
  232. for name in logger_names:
  233. if not name or not isinstance(loggerDict[name],
  234. logging.PlaceHolder):
  235. for handler in logging.getLogger(name).handlers:
  236. handler.createLock()
  237. logging._lock = threading.RLock()
  238. try:
  239. util._finalizer_registry.clear()
  240. util._run_after_forkers()
  241. finally:
  242. # delay finalization of the old process object until after
  243. # _run_after_forkers() is executed
  244. del old_process
  245. util.info('child process %s calling self.run()', self.pid)
  246. try:
  247. self.run()
  248. exitcode = 0
  249. finally:
  250. util._exit_function()
  251. except SystemExit as exc:
  252. if not exc.args:
  253. exitcode = 1
  254. elif isinstance(exc.args[0], int):
  255. exitcode = exc.args[0]
  256. else:
  257. sys.stderr.write(str(exc.args[0]) + '\n')
  258. _maybe_flush(sys.stderr)
  259. exitcode = 0 if isinstance(exc.args[0], str) else 1
  260. except:
  261. exitcode = 1
  262. if not util.error('Process %s', self.name, exc_info=True):
  263. import traceback
  264. sys.stderr.write('Process %s:\n' % self.name)
  265. traceback.print_exc()
  266. finally:
  267. util.info('process %s exiting with exitcode %d',
  268. self.pid, exitcode)
  269. _maybe_flush(sys.stdout)
  270. _maybe_flush(sys.stderr)
  271. return exitcode
  272. #
  273. # We subclass bytes to avoid accidental transmission of auth keys over network
  274. #
  275. class AuthenticationString(bytes):
  276. def __reduce__(self):
  277. from .forking import Popen
  278. if not Popen.thread_is_spawning():
  279. raise TypeError(
  280. 'Pickling an AuthenticationString object is '
  281. 'disallowed for security reasons')
  282. return AuthenticationString, (bytes(self),)
  283. #
  284. # Create object representing the main process
  285. #
  286. class _MainProcess(Process):
  287. def __init__(self):
  288. self._identity = ()
  289. self._daemonic = False
  290. self._name = 'MainProcess'
  291. self._parent_pid = None
  292. self._popen = None
  293. self._counter = itertools.count(1)
  294. self._children = set()
  295. self._authkey = AuthenticationString(os.urandom(32))
  296. self._tempdir = None
  297. self._semprefix = 'mp-' + binascii.hexlify(
  298. os.urandom(4)).decode('ascii')
  299. self._unlinkfd = None
  300. _current_process = _MainProcess()
  301. del _MainProcess
  302. #
  303. # Give names to some return codes
  304. #
  305. _exitcode_to_name = {}
  306. for name, signum in items(signal.__dict__):
  307. if name[:3] == 'SIG' and '_' not in name:
  308. _exitcode_to_name[-signum] = name
  309. _dangling = WeakSet() if WeakSet is not None else None