Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

celeryd_detach.py 5.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181
  1. # -*- coding: utf-8 -*-
  2. """
  3. celery.bin.celeryd_detach
  4. ~~~~~~~~~~~~~~~~~~~~~~~~~
  5. Program used to daemonize the worker
  6. Using :func:`os.execv` because forking and multiprocessing
  7. leads to weird issues (it was a long time ago now, but it
  8. could have something to do with the threading mutex bug)
  9. """
  10. from __future__ import absolute_import
  11. import celery
  12. import os
  13. import sys
  14. from optparse import OptionParser, BadOptionError
  15. from celery.platforms import EX_FAILURE, detached
  16. from celery.utils import default_nodename, node_format
  17. from celery.utils.log import get_logger
  18. from celery.bin.base import daemon_options, Option
  19. __all__ = ['detached_celeryd', 'detach']
  20. logger = get_logger(__name__)
  21. C_FAKEFORK = os.environ.get('C_FAKEFORK')
  22. OPTION_LIST = daemon_options(default_pidfile='celeryd.pid') + (
  23. Option('--workdir', default=None, dest='working_directory'),
  24. Option('-n', '--hostname'),
  25. Option('--fake',
  26. default=False, action='store_true', dest='fake',
  27. help="Don't fork (for debugging purposes)"),
  28. )
  29. def detach(path, argv, logfile=None, pidfile=None, uid=None,
  30. gid=None, umask=None, working_directory=None, fake=False, app=None,
  31. executable=None, hostname=None):
  32. hostname = default_nodename(hostname)
  33. logfile = node_format(logfile, hostname)
  34. pidfile = node_format(pidfile, hostname)
  35. fake = 1 if C_FAKEFORK else fake
  36. with detached(logfile, pidfile, uid, gid, umask, working_directory, fake,
  37. after_forkers=False):
  38. try:
  39. if executable is not None:
  40. path = executable
  41. os.execv(path, [path] + argv)
  42. except Exception:
  43. if app is None:
  44. from celery import current_app
  45. app = current_app
  46. app.log.setup_logging_subsystem(
  47. 'ERROR', logfile, hostname=hostname)
  48. logger.critical("Can't exec %r", ' '.join([path] + argv),
  49. exc_info=True)
  50. return EX_FAILURE
  51. class PartialOptionParser(OptionParser):
  52. def __init__(self, *args, **kwargs):
  53. self.leftovers = []
  54. OptionParser.__init__(self, *args, **kwargs)
  55. def _process_long_opt(self, rargs, values):
  56. arg = rargs.pop(0)
  57. if '=' in arg:
  58. opt, next_arg = arg.split('=', 1)
  59. rargs.insert(0, next_arg)
  60. had_explicit_value = True
  61. else:
  62. opt = arg
  63. had_explicit_value = False
  64. try:
  65. opt = self._match_long_opt(opt)
  66. option = self._long_opt.get(opt)
  67. except BadOptionError:
  68. option = None
  69. if option:
  70. if option.takes_value():
  71. nargs = option.nargs
  72. if len(rargs) < nargs:
  73. if nargs == 1:
  74. self.error('{0} requires an argument'.format(opt))
  75. else:
  76. self.error('{0} requires {1} arguments'.format(
  77. opt, nargs))
  78. elif nargs == 1:
  79. value = rargs.pop(0)
  80. else:
  81. value = tuple(rargs[0:nargs])
  82. del rargs[0:nargs]
  83. elif had_explicit_value:
  84. self.error('{0} option does not take a value'.format(opt))
  85. else:
  86. value = None
  87. option.process(opt, value, values, self)
  88. else:
  89. self.leftovers.append(arg)
  90. def _process_short_opts(self, rargs, values):
  91. arg = rargs[0]
  92. try:
  93. OptionParser._process_short_opts(self, rargs, values)
  94. except BadOptionError:
  95. self.leftovers.append(arg)
  96. if rargs and not rargs[0][0] == '-':
  97. self.leftovers.append(rargs.pop(0))
  98. class detached_celeryd(object):
  99. option_list = OPTION_LIST
  100. usage = '%prog [options] [celeryd options]'
  101. version = celery.VERSION_BANNER
  102. description = ('Detaches Celery worker nodes. See `celery worker --help` '
  103. 'for the list of supported worker arguments.')
  104. command = sys.executable
  105. execv_path = sys.executable
  106. if sys.version_info < (2, 7): # does not support pkg/__main__.py
  107. execv_argv = ['-m', 'celery.__main__', 'worker']
  108. else:
  109. execv_argv = ['-m', 'celery', 'worker']
  110. def __init__(self, app=None):
  111. self.app = app
  112. def Parser(self, prog_name):
  113. return PartialOptionParser(prog=prog_name,
  114. option_list=self.option_list,
  115. usage=self.usage,
  116. description=self.description,
  117. version=self.version)
  118. def parse_options(self, prog_name, argv):
  119. parser = self.Parser(prog_name)
  120. options, values = parser.parse_args(argv)
  121. if options.logfile:
  122. parser.leftovers.append('--logfile={0}'.format(options.logfile))
  123. if options.pidfile:
  124. parser.leftovers.append('--pidfile={0}'.format(options.pidfile))
  125. if options.hostname:
  126. parser.leftovers.append('--hostname={0}'.format(options.hostname))
  127. return options, values, parser.leftovers
  128. def execute_from_commandline(self, argv=None):
  129. if argv is None:
  130. argv = sys.argv
  131. config = []
  132. seen_cargs = 0
  133. for arg in argv:
  134. if seen_cargs:
  135. config.append(arg)
  136. else:
  137. if arg == '--':
  138. seen_cargs = 1
  139. config.append(arg)
  140. prog_name = os.path.basename(argv[0])
  141. options, values, leftovers = self.parse_options(prog_name, argv[1:])
  142. sys.exit(detach(
  143. app=self.app, path=self.execv_path,
  144. argv=self.execv_argv + leftovers + config,
  145. **vars(options)
  146. ))
  147. def main(app=None):
  148. detached_celeryd(app).execute_from_commandline()
  149. if __name__ == '__main__': # pragma: no cover
  150. main()