added celery and tasks but most important added tag_list
This commit is contained in:
Esther Kleinhenz
@@ -0,0 +1,323 @@
|
||||
"""Python multiprocessing fork with improvements and bugfixes"""
|
||||
#
|
||||
# Package analogous to 'threading.py' but using processes
|
||||
#
|
||||
# multiprocessing/__init__.py
|
||||
#
|
||||
# This package is intended to duplicate the functionality (and much of
|
||||
# the API) of threading.py but uses processes instead of threads. A
|
||||
# subpackage 'multiprocessing.dummy' has the same API but is a simple
|
||||
# wrapper for 'threading'.
|
||||
#
|
||||
# Try calling `multiprocessing.doc.main()` to read the html
|
||||
# documentation in a webbrowser.
|
||||
#
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from .exceptions import ( # noqa
|
||||
ProcessError,
|
||||
BufferTooShort,
|
||||
TimeoutError,
|
||||
AuthenticationError,
|
||||
TimeLimitExceeded,
|
||||
SoftTimeLimitExceeded,
|
||||
WorkerLostError,
|
||||
)
|
||||
from .process import Process, current_process, active_children
|
||||
from .util import SUBDEBUG, SUBWARNING
|
||||
|
||||
VERSION = (3, 3, 0, 23)
|
||||
__version__ = '.'.join(map(str, VERSION[0:4])) + "".join(VERSION[4:])
|
||||
__author__ = 'R Oudkerk / Python Software Foundation'
|
||||
__author_email__ = 'python-dev@python.org'
|
||||
__maintainer__ = 'Ask Solem'
|
||||
__contact__ = "ask@celeryproject.org"
|
||||
__homepage__ = "http://github.com/celery/billiard"
|
||||
__docformat__ = "restructuredtext"
|
||||
|
||||
# -eof meta-
|
||||
|
||||
__all__ = [
|
||||
'Process', 'current_process', 'active_children', 'freeze_support',
|
||||
'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger',
|
||||
'allow_connection_pickling', 'BufferTooShort', 'TimeoutError',
|
||||
'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
|
||||
'Event', 'Queue', 'JoinableQueue', 'Pool', 'Value', 'Array',
|
||||
'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', 'set_executable',
|
||||
'forking_enable', 'forking_is_enabled'
|
||||
]
|
||||
|
||||
|
||||
def ensure_multiprocessing():
|
||||
from ._ext import ensure_multiprocessing
|
||||
return ensure_multiprocessing()
|
||||
|
||||
|
||||
W_NO_EXECV = """\
|
||||
force_execv is not supported as the billiard C extension \
|
||||
is not installed\
|
||||
"""
|
||||
|
||||
#
|
||||
# Definitions not depending on native semaphores
|
||||
#
|
||||
|
||||
|
||||
def Manager():
|
||||
'''
|
||||
Returns a manager associated with a running server process
|
||||
|
||||
The managers methods such as `Lock()`, `Condition()` and `Queue()`
|
||||
can be used to create shared objects.
|
||||
'''
|
||||
from .managers import SyncManager
|
||||
m = SyncManager()
|
||||
m.start()
|
||||
return m
|
||||
|
||||
|
||||
def Pipe(duplex=True, rnonblock=False, wnonblock=False):
|
||||
'''
|
||||
Returns two connection object connected by a pipe
|
||||
'''
|
||||
from billiard.connection import Pipe
|
||||
return Pipe(duplex, rnonblock, wnonblock)
|
||||
|
||||
|
||||
def cpu_count():
|
||||
'''
|
||||
Returns the number of CPUs in the system
|
||||
'''
|
||||
if sys.platform == 'win32':
|
||||
try:
|
||||
num = int(os.environ['NUMBER_OF_PROCESSORS'])
|
||||
except (ValueError, KeyError):
|
||||
num = 0
|
||||
elif 'bsd' in sys.platform or sys.platform == 'darwin':
|
||||
comm = '/sbin/sysctl -n hw.ncpu'
|
||||
if sys.platform == 'darwin':
|
||||
comm = '/usr' + comm
|
||||
try:
|
||||
with os.popen(comm) as p:
|
||||
num = int(p.read())
|
||||
except ValueError:
|
||||
num = 0
|
||||
else:
|
||||
try:
|
||||
num = os.sysconf('SC_NPROCESSORS_ONLN')
|
||||
except (ValueError, OSError, AttributeError):
|
||||
num = 0
|
||||
|
||||
if num >= 1:
|
||||
return num
|
||||
else:
|
||||
raise NotImplementedError('cannot determine number of cpus')
|
||||
|
||||
|
||||
def freeze_support():
|
||||
'''
|
||||
Check whether this is a fake forked process in a frozen executable.
|
||||
If so then run code specified by commandline and exit.
|
||||
'''
|
||||
if sys.platform == 'win32' and getattr(sys, 'frozen', False):
|
||||
from .forking import freeze_support
|
||||
freeze_support()
|
||||
|
||||
|
||||
def get_logger():
|
||||
'''
|
||||
Return package logger -- if it does not already exist then it is created
|
||||
'''
|
||||
from .util import get_logger
|
||||
return get_logger()
|
||||
|
||||
|
||||
def log_to_stderr(level=None):
|
||||
'''
|
||||
Turn on logging and add a handler which prints to stderr
|
||||
'''
|
||||
from .util import log_to_stderr
|
||||
return log_to_stderr(level)
|
||||
|
||||
|
||||
def allow_connection_pickling():
|
||||
'''
|
||||
Install support for sending connections and sockets between processes
|
||||
'''
|
||||
from . import reduction # noqa
|
||||
|
||||
#
|
||||
# Definitions depending on native semaphores
|
||||
#
|
||||
|
||||
|
||||
def Lock():
|
||||
'''
|
||||
Returns a non-recursive lock object
|
||||
'''
|
||||
from .synchronize import Lock
|
||||
return Lock()
|
||||
|
||||
|
||||
def RLock():
|
||||
'''
|
||||
Returns a recursive lock object
|
||||
'''
|
||||
from .synchronize import RLock
|
||||
return RLock()
|
||||
|
||||
|
||||
def Condition(lock=None):
|
||||
'''
|
||||
Returns a condition object
|
||||
'''
|
||||
from .synchronize import Condition
|
||||
return Condition(lock)
|
||||
|
||||
|
||||
def Semaphore(value=1):
|
||||
'''
|
||||
Returns a semaphore object
|
||||
'''
|
||||
from .synchronize import Semaphore
|
||||
return Semaphore(value)
|
||||
|
||||
|
||||
def BoundedSemaphore(value=1):
|
||||
'''
|
||||
Returns a bounded semaphore object
|
||||
'''
|
||||
from .synchronize import BoundedSemaphore
|
||||
return BoundedSemaphore(value)
|
||||
|
||||
|
||||
def Event():
|
||||
'''
|
||||
Returns an event object
|
||||
'''
|
||||
from .synchronize import Event
|
||||
return Event()
|
||||
|
||||
|
||||
def Queue(maxsize=0):
|
||||
'''
|
||||
Returns a queue object
|
||||
'''
|
||||
from .queues import Queue
|
||||
return Queue(maxsize)
|
||||
|
||||
|
||||
def JoinableQueue(maxsize=0):
|
||||
'''
|
||||
Returns a queue object
|
||||
'''
|
||||
from .queues import JoinableQueue
|
||||
return JoinableQueue(maxsize)
|
||||
|
||||
|
||||
def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None,
|
||||
timeout=None, soft_timeout=None, lost_worker_timeout=None,
|
||||
max_restarts=None, max_restart_freq=1, on_process_up=None,
|
||||
on_process_down=None, on_timeout_set=None, on_timeout_cancel=None,
|
||||
threads=True, semaphore=None, putlocks=False, allow_restart=False):
|
||||
'''
|
||||
Returns a process pool object
|
||||
'''
|
||||
from .pool import Pool
|
||||
return Pool(processes, initializer, initargs, maxtasksperchild,
|
||||
timeout, soft_timeout, lost_worker_timeout,
|
||||
max_restarts, max_restart_freq, on_process_up,
|
||||
on_process_down, on_timeout_set, on_timeout_cancel,
|
||||
threads, semaphore, putlocks, allow_restart)
|
||||
|
||||
|
||||
def RawValue(typecode_or_type, *args):
|
||||
'''
|
||||
Returns a shared object
|
||||
'''
|
||||
from .sharedctypes import RawValue
|
||||
return RawValue(typecode_or_type, *args)
|
||||
|
||||
|
||||
def RawArray(typecode_or_type, size_or_initializer):
|
||||
'''
|
||||
Returns a shared array
|
||||
'''
|
||||
from .sharedctypes import RawArray
|
||||
return RawArray(typecode_or_type, size_or_initializer)
|
||||
|
||||
|
||||
def Value(typecode_or_type, *args, **kwds):
|
||||
'''
|
||||
Returns a synchronized shared object
|
||||
'''
|
||||
from .sharedctypes import Value
|
||||
return Value(typecode_or_type, *args, **kwds)
|
||||
|
||||
|
||||
def Array(typecode_or_type, size_or_initializer, **kwds):
|
||||
'''
|
||||
Returns a synchronized shared array
|
||||
'''
|
||||
from .sharedctypes import Array
|
||||
return Array(typecode_or_type, size_or_initializer, **kwds)
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
|
||||
def set_executable(executable):
|
||||
'''
|
||||
Sets the path to a python.exe or pythonw.exe binary used to run
|
||||
child processes on Windows instead of sys.executable.
|
||||
Useful for people embedding Python.
|
||||
'''
|
||||
from .forking import set_executable
|
||||
set_executable(executable)
|
||||
|
||||
|
||||
def forking_is_enabled():
|
||||
'''
|
||||
Returns a boolean value indicating whether billiard is
|
||||
currently set to create child processes by forking the current
|
||||
python process rather than by starting a new instances of python.
|
||||
|
||||
On Windows this always returns `False`. On Unix it returns `True` by
|
||||
default.
|
||||
'''
|
||||
from . import forking
|
||||
return forking._forking_is_enabled
|
||||
|
||||
|
||||
def forking_enable(value):
|
||||
'''
|
||||
Enable/disable creation of child process by forking the current process.
|
||||
|
||||
`value` should be a boolean value. If `value` is true then
|
||||
forking is enabled. If `value` is false then forking is disabled.
|
||||
On systems with `os.fork()` forking is enabled by default, and on
|
||||
other systems it is always disabled.
|
||||
'''
|
||||
if not value:
|
||||
from ._ext import supports_exec
|
||||
if supports_exec:
|
||||
from . import forking
|
||||
if value and not hasattr(os, 'fork'):
|
||||
raise ValueError('os.fork() not found')
|
||||
forking._forking_is_enabled = bool(value)
|
||||
if not value:
|
||||
os.environ["MULTIPROCESSING_FORKING_DISABLE"] = "1"
|
||||
else:
|
||||
warnings.warn(RuntimeWarning(W_NO_EXECV))
|
||||
if os.environ.get("MULTIPROCESSING_FORKING_DISABLE"):
|
||||
forking_enable(False)
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
|
||||
supports_exec = True
|
||||
|
||||
from .compat import _winapi as win32 # noqa
|
||||
|
||||
if sys.platform.startswith("java"):
|
||||
_billiard = None
|
||||
else:
|
||||
try:
|
||||
import _billiard # noqa
|
||||
except ImportError:
|
||||
import _multiprocessing as _billiard # noqa
|
||||
supports_exec = False
|
||||
try:
|
||||
Connection = _billiard.Connection
|
||||
except AttributeError: # Py3
|
||||
from billiard.connection import Connection # noqa
|
||||
|
||||
PipeConnection = getattr(_billiard, "PipeConnection", None)
|
||||
|
||||
|
||||
def ensure_multiprocessing():
|
||||
if _billiard is None:
|
||||
raise NotImplementedError("multiprocessing not supported")
|
||||
|
||||
|
||||
def ensure_SemLock():
|
||||
try:
|
||||
from _billiard import SemLock # noqa
|
||||
except ImportError:
|
||||
try:
|
||||
from _multiprocessing import SemLock # noqa
|
||||
except ImportError:
|
||||
raise ImportError("""\
|
||||
This platform lacks a functioning sem_open implementation, therefore,
|
||||
the required synchronization primitives needed will not function,
|
||||
see issue 3770.""")
|
||||
@@ -0,0 +1,116 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
billiard._win
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
Windows utilities to terminate process groups.
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
|
||||
# psutil is painfully slow in win32. So to avoid adding big
|
||||
# dependencies like pywin32 a ctypes based solution is preferred
|
||||
|
||||
# Code based on the winappdbg project http://winappdbg.sourceforge.net/
|
||||
# (BSD License)
|
||||
from ctypes import (
|
||||
byref, sizeof, windll,
|
||||
Structure, WinError, POINTER,
|
||||
c_size_t, c_char, c_void_p,
|
||||
)
|
||||
from ctypes.wintypes import DWORD, LONG
|
||||
|
||||
ERROR_NO_MORE_FILES = 18
|
||||
INVALID_HANDLE_VALUE = c_void_p(-1).value
|
||||
|
||||
|
||||
class PROCESSENTRY32(Structure):
|
||||
_fields_ = [
|
||||
('dwSize', DWORD),
|
||||
('cntUsage', DWORD),
|
||||
('th32ProcessID', DWORD),
|
||||
('th32DefaultHeapID', c_size_t),
|
||||
('th32ModuleID', DWORD),
|
||||
('cntThreads', DWORD),
|
||||
('th32ParentProcessID', DWORD),
|
||||
('pcPriClassBase', LONG),
|
||||
('dwFlags', DWORD),
|
||||
('szExeFile', c_char * 260),
|
||||
]
|
||||
LPPROCESSENTRY32 = POINTER(PROCESSENTRY32)
|
||||
|
||||
|
||||
def CreateToolhelp32Snapshot(dwFlags=2, th32ProcessID=0):
|
||||
hSnapshot = windll.kernel32.CreateToolhelp32Snapshot(dwFlags,
|
||||
th32ProcessID)
|
||||
if hSnapshot == INVALID_HANDLE_VALUE:
|
||||
raise WinError()
|
||||
return hSnapshot
|
||||
|
||||
|
||||
def Process32First(hSnapshot, pe=None):
|
||||
return _Process32n(windll.kernel32.Process32First, hSnapshot, pe)
|
||||
|
||||
|
||||
def Process32Next(hSnapshot, pe=None):
|
||||
return _Process32n(windll.kernel32.Process32Next, hSnapshot, pe)
|
||||
|
||||
|
||||
def _Process32n(fun, hSnapshot, pe=None):
|
||||
if pe is None:
|
||||
pe = PROCESSENTRY32()
|
||||
pe.dwSize = sizeof(PROCESSENTRY32)
|
||||
success = fun(hSnapshot, byref(pe))
|
||||
if not success:
|
||||
if windll.kernel32.GetLastError() == ERROR_NO_MORE_FILES:
|
||||
return
|
||||
raise WinError()
|
||||
return pe
|
||||
|
||||
|
||||
def get_all_processes_pids():
|
||||
"""Return a dictionary with all processes pids as keys and their
|
||||
parents as value. Ignore processes with no parents.
|
||||
"""
|
||||
h = CreateToolhelp32Snapshot()
|
||||
parents = {}
|
||||
pe = Process32First(h)
|
||||
while pe:
|
||||
if pe.th32ParentProcessID:
|
||||
parents[pe.th32ProcessID] = pe.th32ParentProcessID
|
||||
pe = Process32Next(h, pe)
|
||||
|
||||
return parents
|
||||
|
||||
|
||||
def get_processtree_pids(pid, include_parent=True):
|
||||
"""Return a list with all the pids of a process tree"""
|
||||
parents = get_all_processes_pids()
|
||||
all_pids = list(parents.keys())
|
||||
pids = set([pid])
|
||||
while 1:
|
||||
pids_new = pids.copy()
|
||||
|
||||
for _pid in all_pids:
|
||||
if parents[_pid] in pids:
|
||||
pids_new.add(_pid)
|
||||
|
||||
if pids_new == pids:
|
||||
break
|
||||
|
||||
pids = pids_new.copy()
|
||||
|
||||
if not include_parent:
|
||||
pids.remove(pid)
|
||||
|
||||
return list(pids)
|
||||
|
||||
|
||||
def kill_processtree(pid, signum):
|
||||
"""Kill a process and all its descendants"""
|
||||
family_pids = get_processtree_pids(pid)
|
||||
|
||||
for _pid in family_pids:
|
||||
os.kill(_pid, signum)
|
||||
@@ -0,0 +1,134 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
This module contains utilities added by billiard, to keep
|
||||
"non-core" functionality out of ``.util``."""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import pickle as pypickle
|
||||
try:
|
||||
import cPickle as cpickle
|
||||
except ImportError: # pragma: no cover
|
||||
cpickle = None # noqa
|
||||
|
||||
from .exceptions import RestartFreqExceeded
|
||||
from .five import monotonic
|
||||
|
||||
if sys.version_info < (2, 6): # pragma: no cover
|
||||
# cPickle does not use absolute_imports
|
||||
pickle = pypickle
|
||||
pickle_load = pypickle.load
|
||||
pickle_loads = pypickle.loads
|
||||
else:
|
||||
pickle = cpickle or pypickle
|
||||
pickle_load = pickle.load
|
||||
pickle_loads = pickle.loads
|
||||
|
||||
# cPickle.loads does not support buffer() objects,
|
||||
# but we can just create a StringIO and use load.
|
||||
if sys.version_info[0] == 3:
|
||||
from io import BytesIO
|
||||
else:
|
||||
try:
|
||||
from cStringIO import StringIO as BytesIO # noqa
|
||||
except ImportError:
|
||||
from StringIO import StringIO as BytesIO # noqa
|
||||
|
||||
EX_SOFTWARE = 70
|
||||
|
||||
TERMSIGS_DEFAULT = (
|
||||
'SIGHUP',
|
||||
'SIGQUIT',
|
||||
'SIGTERM',
|
||||
'SIGUSR1',
|
||||
'SIGUSR2'
|
||||
)
|
||||
|
||||
TERMSIGS_FULL = (
|
||||
'SIGHUP',
|
||||
'SIGQUIT',
|
||||
'SIGTRAP',
|
||||
'SIGABRT',
|
||||
'SIGEMT',
|
||||
'SIGSYS',
|
||||
'SIGPIPE',
|
||||
'SIGALRM',
|
||||
'SIGTERM',
|
||||
'SIGXCPU',
|
||||
'SIGXFSZ',
|
||||
'SIGVTALRM',
|
||||
'SIGPROF',
|
||||
'SIGUSR1',
|
||||
'SIGUSR2',
|
||||
)
|
||||
|
||||
#: set by signal handlers just before calling exit.
|
||||
#: if this is true after the sighandler returns it means that something
|
||||
#: went wrong while terminating the process, and :func:`os._exit`
|
||||
#: must be called ASAP.
|
||||
_should_have_exited = [False]
|
||||
|
||||
|
||||
def pickle_loads(s, load=pickle_load):
|
||||
# used to support buffer objects
|
||||
return load(BytesIO(s))
|
||||
|
||||
|
||||
def maybe_setsignal(signum, handler):
|
||||
try:
|
||||
signal.signal(signum, handler)
|
||||
except (OSError, AttributeError, ValueError, RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _shutdown_cleanup(signum, frame):
|
||||
# we will exit here so if the signal is received a second time
|
||||
# we can be sure that something is very wrong and we may be in
|
||||
# a crashing loop.
|
||||
if _should_have_exited[0]:
|
||||
os._exit(EX_SOFTWARE)
|
||||
maybe_setsignal(signum, signal.SIG_DFL)
|
||||
_should_have_exited[0] = True
|
||||
sys.exit(-(256 - signum))
|
||||
|
||||
|
||||
def reset_signals(handler=_shutdown_cleanup, full=False):
|
||||
for sig in TERMSIGS_FULL if full else TERMSIGS_DEFAULT:
|
||||
try:
|
||||
signum = getattr(signal, sig)
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
current = signal.getsignal(signum)
|
||||
if current is not None and current != signal.SIG_IGN:
|
||||
maybe_setsignal(signum, handler)
|
||||
|
||||
|
||||
class restart_state(object):
|
||||
RestartFreqExceeded = RestartFreqExceeded
|
||||
|
||||
def __init__(self, maxR, maxT):
|
||||
self.maxR, self.maxT = maxR, maxT
|
||||
self.R, self.T = 0, None
|
||||
|
||||
def step(self, now=None):
|
||||
now = monotonic() if now is None else now
|
||||
R = self.R
|
||||
if self.T and now - self.T >= self.maxT:
|
||||
# maxT passed, reset counter and time passed.
|
||||
self.T, self.R = now, 0
|
||||
elif self.maxR and self.R >= self.maxR:
|
||||
# verify that R has a value as the result handler
|
||||
# resets this when a job is accepted. If a job is accepted
|
||||
# the startup probably went fine (startup restart burst
|
||||
# protection)
|
||||
if self.R: # pragma: no cover
|
||||
self.R = 0 # reset in case someone catches the error
|
||||
raise self.RestartFreqExceeded("%r in %rs" % (R, self.maxT))
|
||||
# first run sets T
|
||||
if self.T is None:
|
||||
self.T = now
|
||||
self.R += 1
|
||||
@@ -0,0 +1,107 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import errno
|
||||
import os
|
||||
import sys
|
||||
|
||||
from .five import range
|
||||
|
||||
if sys.platform == 'win32':
|
||||
try:
|
||||
import _winapi # noqa
|
||||
except ImportError: # pragma: no cover
|
||||
try:
|
||||
from _billiard import win32 as _winapi # noqa
|
||||
except (ImportError, AttributeError):
|
||||
from _multiprocessing import win32 as _winapi # noqa
|
||||
else:
|
||||
_winapi = None # noqa
|
||||
|
||||
|
||||
if sys.version_info > (2, 7, 5):
|
||||
buf_t, is_new_buffer = memoryview, True # noqa
|
||||
else:
|
||||
buf_t, is_new_buffer = buffer, False # noqa
|
||||
|
||||
if hasattr(os, 'write'):
|
||||
__write__ = os.write
|
||||
|
||||
if is_new_buffer:
|
||||
|
||||
def send_offset(fd, buf, offset):
|
||||
return __write__(fd, buf[offset:])
|
||||
|
||||
else: # Py<2.7.6
|
||||
|
||||
def send_offset(fd, buf, offset): # noqa
|
||||
return __write__(fd, buf_t(buf, offset))
|
||||
|
||||
else: # non-posix platform
|
||||
|
||||
def send_offset(fd, buf, offset): # noqa
|
||||
raise NotImplementedError('send_offset')
|
||||
|
||||
|
||||
if sys.version_info[0] == 3:
|
||||
bytes = bytes
|
||||
else:
|
||||
_bytes = bytes
|
||||
|
||||
# the 'bytes' alias in Python2 does not support an encoding argument.
|
||||
|
||||
class bytes(_bytes): # noqa
|
||||
|
||||
def __new__(cls, *args):
|
||||
if len(args) > 1:
|
||||
return _bytes(args[0]).encode(*args[1:])
|
||||
return _bytes(*args)
|
||||
|
||||
try:
|
||||
closerange = os.closerange
|
||||
except AttributeError:
|
||||
|
||||
def closerange(fd_low, fd_high): # noqa
|
||||
for fd in reversed(range(fd_low, fd_high)):
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EBADF:
|
||||
raise
|
||||
|
||||
|
||||
def get_errno(exc):
|
||||
""":exc:`socket.error` and :exc:`IOError` first got
|
||||
the ``.errno`` attribute in Py2.7"""
|
||||
try:
|
||||
return exc.errno
|
||||
except AttributeError:
|
||||
try:
|
||||
# e.args = (errno, reason)
|
||||
if isinstance(exc.args, tuple) and len(exc.args) == 2:
|
||||
return exc.args[0]
|
||||
except AttributeError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if sys.platform == 'win32':
|
||||
|
||||
def setblocking(handle, blocking):
|
||||
raise NotImplementedError('setblocking not implemented on win32')
|
||||
|
||||
def isblocking(handle):
|
||||
raise NotImplementedError('isblocking not implemented on win32')
|
||||
|
||||
else:
|
||||
from os import O_NONBLOCK
|
||||
from fcntl import fcntl, F_GETFL, F_SETFL
|
||||
|
||||
def isblocking(handle): # noqa
|
||||
return not (fcntl(handle, F_GETFL) & O_NONBLOCK)
|
||||
|
||||
def setblocking(handle, blocking): # noqa
|
||||
flags = fcntl(handle, F_GETFL, 0)
|
||||
fcntl(
|
||||
handle, F_SETFL,
|
||||
flags & (~O_NONBLOCK) if blocking else flags | O_NONBLOCK,
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
|
||||
is_pypy = hasattr(sys, 'pypy_version_info')
|
||||
|
||||
if sys.version_info[0] == 3:
|
||||
from .py3 import connection
|
||||
else:
|
||||
from .py2 import connection # noqa
|
||||
|
||||
|
||||
if is_pypy:
|
||||
import _multiprocessing
|
||||
from .compat import setblocking, send_offset
|
||||
|
||||
class Connection(_multiprocessing.Connection):
|
||||
|
||||
def send_offset(self, buf, offset):
|
||||
return send_offset(self.fileno(), buf, offset)
|
||||
|
||||
def setblocking(self, blocking):
|
||||
setblocking(self.fileno(), blocking)
|
||||
_multiprocessing.Connection = Connection
|
||||
|
||||
|
||||
sys.modules[__name__] = connection
|
||||
@@ -0,0 +1,165 @@
|
||||
#
|
||||
# Support for the API of the multiprocessing package using threads
|
||||
#
|
||||
# multiprocessing/dummy/__init__.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
# 3. Neither the name of author nor the names of any contributors may be
|
||||
# used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
# SUCH DAMAGE.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
#
|
||||
# Imports
|
||||
#
|
||||
|
||||
import threading
|
||||
import sys
|
||||
import weakref
|
||||
import array
|
||||
|
||||
from threading import Lock, RLock, Semaphore, BoundedSemaphore
|
||||
from threading import Event
|
||||
|
||||
from billiard.five import Queue
|
||||
|
||||
from billiard.connection import Pipe
|
||||
|
||||
__all__ = [
|
||||
'Process', 'current_process', 'active_children', 'freeze_support',
|
||||
'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
|
||||
'Event', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue'
|
||||
]
|
||||
|
||||
|
||||
class DummyProcess(threading.Thread):
|
||||
|
||||
def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
|
||||
threading.Thread.__init__(self, group, target, name, args, kwargs)
|
||||
self._pid = None
|
||||
self._children = weakref.WeakKeyDictionary()
|
||||
self._start_called = False
|
||||
self._parent = current_process()
|
||||
|
||||
def start(self):
|
||||
assert self._parent is current_process()
|
||||
self._start_called = True
|
||||
self._parent._children[self] = None
|
||||
threading.Thread.start(self)
|
||||
|
||||
@property
|
||||
def exitcode(self):
|
||||
if self._start_called and not self.is_alive():
|
||||
return 0
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
_Condition = threading._Condition
|
||||
except AttributeError: # Py3
|
||||
_Condition = threading.Condition # noqa
|
||||
|
||||
|
||||
class Condition(_Condition):
|
||||
if sys.version_info[0] == 3:
|
||||
notify_all = _Condition.notifyAll
|
||||
else:
|
||||
notify_all = _Condition.notifyAll.__func__
|
||||
|
||||
|
||||
Process = DummyProcess
|
||||
current_process = threading.currentThread
|
||||
current_process()._children = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def active_children():
|
||||
children = current_process()._children
|
||||
for p in list(children):
|
||||
if not p.is_alive():
|
||||
children.pop(p, None)
|
||||
return list(children)
|
||||
|
||||
|
||||
def freeze_support():
|
||||
pass
|
||||
|
||||
|
||||
class Namespace(object):
|
||||
|
||||
def __init__(self, **kwds):
|
||||
self.__dict__.update(kwds)
|
||||
|
||||
def __repr__(self):
|
||||
items = list(self.__dict__.items())
|
||||
temp = []
|
||||
for name, value in items:
|
||||
if not name.startswith('_'):
|
||||
temp.append('%s=%r' % (name, value))
|
||||
temp.sort()
|
||||
return 'Namespace(%s)' % str.join(', ', temp)
|
||||
|
||||
|
||||
dict = dict
|
||||
list = list
|
||||
|
||||
|
||||
def Array(typecode, sequence, lock=True):
|
||||
return array.array(typecode, sequence)
|
||||
|
||||
|
||||
class Value(object):
|
||||
|
||||
def __init__(self, typecode, value, lock=True):
|
||||
self._typecode = typecode
|
||||
self._value = value
|
||||
|
||||
def _get(self):
|
||||
return self._value
|
||||
|
||||
def _set(self, value):
|
||||
self._value = value
|
||||
value = property(_get, _set)
|
||||
|
||||
def __repr__(self):
|
||||
return '<%r(%r, %r)>' % (type(self).__name__,
|
||||
self._typecode, self._value)
|
||||
|
||||
|
||||
def Manager():
|
||||
return sys.modules[__name__]
|
||||
|
||||
|
||||
def shutdown():
|
||||
pass
|
||||
|
||||
|
||||
def Pool(processes=None, initializer=None, initargs=()):
|
||||
from billiard.pool import ThreadPool
|
||||
return ThreadPool(processes, initializer, initargs)
|
||||
|
||||
JoinableQueue = Queue
|
||||
@@ -0,0 +1,93 @@
|
||||
#
|
||||
# Analogue of `multiprocessing.connection` which uses queues instead of sockets
|
||||
#
|
||||
# multiprocessing/dummy/connection.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
# 3. Neither the name of author nor the names of any contributors may be
|
||||
# used to endorse or promote products derived from this software
|
||||
# without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
# SUCH DAMAGE.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
from billiard.five import Queue
|
||||
|
||||
__all__ = ['Client', 'Listener', 'Pipe']
|
||||
|
||||
families = [None]
|
||||
|
||||
|
||||
class Listener(object):
|
||||
|
||||
def __init__(self, address=None, family=None, backlog=1):
|
||||
self._backlog_queue = Queue(backlog)
|
||||
|
||||
def accept(self):
|
||||
return Connection(*self._backlog_queue.get())
|
||||
|
||||
def close(self):
|
||||
self._backlog_queue = None
|
||||
|
||||
address = property(lambda self: self._backlog_queue)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
self.close()
|
||||
|
||||
|
||||
def Client(address):
|
||||
_in, _out = Queue(), Queue()
|
||||
address.put((_out, _in))
|
||||
return Connection(_in, _out)
|
||||
|
||||
|
||||
def Pipe(duplex=True):
|
||||
a, b = Queue(), Queue()
|
||||
return Connection(a, b), Connection(b, a)
|
||||
|
||||
|
||||
class Connection(object):
|
||||
|
||||
def __init__(self, _in, _out):
|
||||
self._out = _out
|
||||
self._in = _in
|
||||
self.send = self.send_bytes = _out.put
|
||||
self.recv = self.recv_bytes = _in.get
|
||||
|
||||
def poll(self, timeout=0.0):
|
||||
if self._in.qsize() > 0:
|
||||
return True
|
||||
if timeout <= 0.0:
|
||||
return False
|
||||
self._in.not_empty.acquire()
|
||||
self._in.not_empty.wait(timeout)
|
||||
self._in.not_empty.release()
|
||||
return self._in.qsize() > 0
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
__all__ = ['ExceptionInfo', 'Traceback']
|
||||
|
||||
DEFAULT_MAX_FRAMES = sys.getrecursionlimit() // 8
|
||||
|
||||
|
||||
class _Code(object):
|
||||
|
||||
def __init__(self, code):
|
||||
self.co_filename = code.co_filename
|
||||
self.co_name = code.co_name
|
||||
self.co_argcount = code.co_argcount
|
||||
self.co_cellvars = ()
|
||||
self.co_firstlineno = code.co_firstlineno
|
||||
self.co_flags = code.co_flags
|
||||
self.co_freevars = ()
|
||||
self.co_code = b''
|
||||
self.co_lnotab = b''
|
||||
self.co_names = code.co_names
|
||||
self.co_nlocals = code.co_nlocals
|
||||
self.co_stacksize = code.co_stacksize
|
||||
self.co_varnames = ()
|
||||
|
||||
|
||||
class _Frame(object):
|
||||
Code = _Code
|
||||
|
||||
def __init__(self, frame):
|
||||
self.f_builtins = {}
|
||||
self.f_globals = {
|
||||
"__file__": frame.f_globals.get("__file__", "__main__"),
|
||||
"__name__": frame.f_globals.get("__name__"),
|
||||
"__loader__": None,
|
||||
}
|
||||
self.f_locals = fl = {}
|
||||
try:
|
||||
fl["__traceback_hide__"] = frame.f_locals["__traceback_hide__"]
|
||||
except KeyError:
|
||||
pass
|
||||
self.f_trace = None
|
||||
self.f_exc_traceback = None
|
||||
self.f_exc_type = None
|
||||
self.f_exc_value = None
|
||||
self.f_code = self.Code(frame.f_code)
|
||||
self.f_lineno = frame.f_lineno
|
||||
self.f_lasti = frame.f_lasti
|
||||
# don't want to hit https://bugs.python.org/issue21967
|
||||
self.f_restricted = False
|
||||
|
||||
|
||||
class _Object(object):
|
||||
|
||||
def __init__(self, **kw):
|
||||
[setattr(self, k, v) for k, v in kw.items()]
|
||||
|
||||
|
||||
class _Truncated(object):
|
||||
|
||||
def __init__(self):
|
||||
self.tb_lineno = -1
|
||||
self.tb_frame = _Object(
|
||||
f_globals={"__file__": "",
|
||||
"__name__": "",
|
||||
"__loader__": None},
|
||||
f_fileno=None,
|
||||
f_code=_Object(co_filename="...",
|
||||
co_name="[rest of traceback truncated]"),
|
||||
)
|
||||
self.tb_next = None
|
||||
self.tb_lasti = 0
|
||||
|
||||
|
||||
class Traceback(object):
|
||||
Frame = _Frame
|
||||
|
||||
def __init__(self, tb, max_frames=DEFAULT_MAX_FRAMES, depth=0):
|
||||
self.tb_frame = self.Frame(tb.tb_frame)
|
||||
self.tb_lineno = tb.tb_lineno
|
||||
self.tb_lasti = tb.tb_lasti
|
||||
self.tb_next = None
|
||||
if tb.tb_next is not None:
|
||||
if depth <= max_frames:
|
||||
self.tb_next = Traceback(tb.tb_next, max_frames, depth + 1)
|
||||
else:
|
||||
self.tb_next = _Truncated()
|
||||
|
||||
|
||||
class ExceptionInfo(object):
|
||||
"""Exception wrapping an exception and its traceback.
|
||||
|
||||
:param exc_info: The exception info tuple as returned by
|
||||
:func:`sys.exc_info`.
|
||||
|
||||
"""
|
||||
|
||||
#: Exception type.
|
||||
type = None
|
||||
|
||||
#: Exception instance.
|
||||
exception = None
|
||||
|
||||
#: Pickleable traceback instance for use with :mod:`traceback`
|
||||
tb = None
|
||||
|
||||
#: String representation of the traceback.
|
||||
traceback = None
|
||||
|
||||
#: Set to true if this is an internal error.
|
||||
internal = False
|
||||
|
||||
def __init__(self, exc_info=None, internal=False):
|
||||
self.type, self.exception, tb = exc_info or sys.exc_info()
|
||||
try:
|
||||
self.tb = Traceback(tb)
|
||||
self.traceback = ''.join(
|
||||
traceback.format_exception(self.type, self.exception, tb),
|
||||
)
|
||||
self.internal = internal
|
||||
finally:
|
||||
del(tb)
|
||||
|
||||
def __str__(self):
|
||||
return self.traceback
|
||||
|
||||
def __repr__(self):
|
||||
return "<ExceptionInfo: %r>" % (self.exception, )
|
||||
|
||||
@property
|
||||
def exc_info(self):
|
||||
return self.type, self.exception, self.tb
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
try:
|
||||
from multiprocessing import (
|
||||
ProcessError,
|
||||
BufferTooShort,
|
||||
TimeoutError,
|
||||
AuthenticationError,
|
||||
)
|
||||
except ImportError:
|
||||
class ProcessError(Exception): # noqa
|
||||
pass
|
||||
|
||||
class BufferTooShort(Exception): # noqa
|
||||
pass
|
||||
|
||||
class TimeoutError(Exception): # noqa
|
||||
pass
|
||||
|
||||
class AuthenticationError(Exception): # noqa
|
||||
pass
|
||||
|
||||
|
||||
class TimeLimitExceeded(Exception):
|
||||
"""The time limit has been exceeded and the job has been terminated."""
|
||||
|
||||
def __str__(self):
|
||||
return "TimeLimitExceeded%s" % (self.args, )
|
||||
|
||||
|
||||
class SoftTimeLimitExceeded(Exception):
|
||||
"""The soft time limit has been exceeded. This exception is raised
|
||||
to give the task a chance to clean up."""
|
||||
|
||||
def __str__(self):
|
||||
return "SoftTimeLimitExceeded%s" % (self.args, )
|
||||
|
||||
|
||||
class WorkerLostError(Exception):
|
||||
"""The worker processing a job has exited prematurely."""
|
||||
|
||||
|
||||
class Terminated(Exception):
|
||||
"""The worker processing a job has been terminated by user request."""
|
||||
|
||||
|
||||
class RestartFreqExceeded(Exception):
|
||||
"""Restarts too fast."""
|
||||
|
||||
|
||||
class CoroStop(Exception):
|
||||
"""Coroutine exit, as opposed to StopIteration which may
|
||||
mean it should be restarted."""
|
||||
pass
|
||||
@@ -0,0 +1,192 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
celery.five
|
||||
~~~~~~~~~~~
|
||||
|
||||
Compatibility implementations of features
|
||||
only available in newer Python versions.
|
||||
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
# ############# py3k #########################################################
|
||||
import sys
|
||||
PY3 = sys.version_info[0] == 3
|
||||
|
||||
try:
|
||||
reload = reload # noqa
|
||||
except NameError: # pragma: no cover
|
||||
from imp import reload # noqa
|
||||
|
||||
try:
|
||||
from UserList import UserList # noqa
|
||||
except ImportError: # pragma: no cover
|
||||
from collections import UserList # noqa
|
||||
|
||||
try:
|
||||
from UserDict import UserDict # noqa
|
||||
except ImportError: # pragma: no cover
|
||||
from collections import UserDict # noqa
|
||||
|
||||
# ############# time.monotonic ###############################################
|
||||
|
||||
if sys.version_info < (3, 3):
|
||||
|
||||
import platform
|
||||
SYSTEM = platform.system()
|
||||
|
||||
try:
|
||||
import ctypes
|
||||
except ImportError: # pragma: no cover
|
||||
ctypes = None # noqa
|
||||
|
||||
if SYSTEM == 'Darwin' and ctypes is not None:
|
||||
from ctypes.util import find_library
|
||||
libSystem = ctypes.CDLL(find_library('libSystem.dylib'))
|
||||
CoreServices = ctypes.CDLL(find_library('CoreServices'),
|
||||
use_errno=True)
|
||||
mach_absolute_time = libSystem.mach_absolute_time
|
||||
mach_absolute_time.restype = ctypes.c_uint64
|
||||
absolute_to_nanoseconds = CoreServices.AbsoluteToNanoseconds
|
||||
absolute_to_nanoseconds.restype = ctypes.c_uint64
|
||||
absolute_to_nanoseconds.argtypes = [ctypes.c_uint64]
|
||||
|
||||
def _monotonic():
|
||||
return absolute_to_nanoseconds(mach_absolute_time()) * 1e-9
|
||||
|
||||
elif SYSTEM == 'Linux' and ctypes is not None:
|
||||
# from stackoverflow:
|
||||
# questions/1205722/how-do-i-get-monotonic-time-durations-in-python
|
||||
import ctypes
|
||||
import os
|
||||
|
||||
CLOCK_MONOTONIC = 1 # see <linux/time.h>
|
||||
|
||||
class timespec(ctypes.Structure):
|
||||
_fields_ = [
|
||||
('tv_sec', ctypes.c_long),
|
||||
('tv_nsec', ctypes.c_long),
|
||||
]
|
||||
|
||||
librt = ctypes.CDLL('librt.so.1', use_errno=True)
|
||||
clock_gettime = librt.clock_gettime
|
||||
clock_gettime.argtypes = [
|
||||
ctypes.c_int, ctypes.POINTER(timespec),
|
||||
]
|
||||
|
||||
def _monotonic(): # noqa
|
||||
t = timespec()
|
||||
if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
|
||||
errno_ = ctypes.get_errno()
|
||||
raise OSError(errno_, os.strerror(errno_))
|
||||
return t.tv_sec + t.tv_nsec * 1e-9
|
||||
else:
|
||||
from time import time as _monotonic
|
||||
|
||||
try:
|
||||
from time import monotonic
|
||||
except ImportError:
|
||||
monotonic = _monotonic # noqa
|
||||
|
||||
if PY3:
|
||||
import builtins
|
||||
|
||||
from queue import Queue, Empty, Full
|
||||
from itertools import zip_longest
|
||||
from io import StringIO, BytesIO
|
||||
|
||||
map = map
|
||||
string = str
|
||||
string_t = str
|
||||
long_t = int
|
||||
text_t = str
|
||||
range = range
|
||||
int_types = (int, )
|
||||
|
||||
open_fqdn = 'builtins.open'
|
||||
|
||||
def items(d):
|
||||
return d.items()
|
||||
|
||||
def keys(d):
|
||||
return d.keys()
|
||||
|
||||
def values(d):
|
||||
return d.values()
|
||||
|
||||
def nextfun(it):
|
||||
return it.__next__
|
||||
|
||||
exec_ = getattr(builtins, 'exec')
|
||||
|
||||
def reraise(tp, value, tb=None):
|
||||
if value.__traceback__ is not tb:
|
||||
raise value.with_traceback(tb)
|
||||
raise value
|
||||
|
||||
class WhateverIO(StringIO):
|
||||
|
||||
def write(self, data):
|
||||
if isinstance(data, bytes):
|
||||
data = data.encode()
|
||||
StringIO.write(self, data)
|
||||
|
||||
else:
|
||||
import __builtin__ as builtins # noqa
|
||||
from Queue import Queue, Empty, Full # noqa
|
||||
from itertools import imap as map, izip_longest as zip_longest # noqa
|
||||
from StringIO import StringIO # noqa
|
||||
string = unicode # noqa
|
||||
string_t = basestring # noqa
|
||||
text_t = unicode
|
||||
long_t = long # noqa
|
||||
range = xrange
|
||||
int_types = (int, long)
|
||||
|
||||
open_fqdn = '__builtin__.open'
|
||||
|
||||
def items(d): # noqa
|
||||
return d.iteritems()
|
||||
|
||||
def keys(d): # noqa
|
||||
return d.iterkeys()
|
||||
|
||||
def values(d): # noqa
|
||||
return d.itervalues()
|
||||
|
||||
def nextfun(it): # noqa
|
||||
return it.next
|
||||
|
||||
def exec_(code, globs=None, locs=None):
|
||||
"""Execute code in a namespace."""
|
||||
if globs is None:
|
||||
frame = sys._getframe(1)
|
||||
globs = frame.f_globals
|
||||
if locs is None:
|
||||
locs = frame.f_locals
|
||||
del frame
|
||||
elif locs is None:
|
||||
locs = globs
|
||||
exec("""exec code in globs, locs""")
|
||||
|
||||
exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")
|
||||
|
||||
BytesIO = WhateverIO = StringIO # noqa
|
||||
|
||||
|
||||
def with_metaclass(Type, skip_attrs=set(['__dict__', '__weakref__'])):
|
||||
"""Class decorator to set metaclass.
|
||||
|
||||
Works with both Python 2 and Python 3 and it does not add
|
||||
an extra class in the lookup order like ``six.with_metaclass`` does
|
||||
(that is -- it copies the original class instead of using inheritance).
|
||||
|
||||
"""
|
||||
|
||||
def _clone_with_metaclass(Class):
|
||||
attrs = dict((key, value) for key, value in items(vars(Class))
|
||||
if key not in skip_attrs)
|
||||
return Type(Class.__name__, Class.__bases__, attrs)
|
||||
|
||||
return _clone_with_metaclass
|
||||
@@ -0,0 +1,580 @@
|
||||
#
|
||||
# Module for starting a process object using os.fork() or CreateProcess()
|
||||
#
|
||||
# multiprocessing/forking.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import warnings
|
||||
|
||||
from pickle import load, HIGHEST_PROTOCOL
|
||||
from billiard import util
|
||||
from billiard import process
|
||||
from billiard.five import int_types
|
||||
from .reduction import dump
|
||||
from .compat import _winapi as win32
|
||||
|
||||
__all__ = ['Popen', 'assert_spawning', 'exit',
|
||||
'duplicate', 'close']
|
||||
|
||||
try:
|
||||
WindowsError = WindowsError # noqa
|
||||
except NameError:
|
||||
class WindowsError(Exception): # noqa
|
||||
pass
|
||||
|
||||
W_OLD_DJANGO_LAYOUT = """\
|
||||
Will add directory %r to path! This is necessary to accommodate \
|
||||
pre-Django 1.4 layouts using setup_environ.
|
||||
You can skip this warning by adding a DJANGO_SETTINGS_MODULE=settings \
|
||||
environment variable.
|
||||
"""
|
||||
|
||||
#
|
||||
# Choose whether to do a fork or spawn (fork+exec) on Unix.
|
||||
# This affects how some shared resources should be created.
|
||||
#
|
||||
|
||||
_forking_is_enabled = sys.platform != 'win32'
|
||||
|
||||
#
|
||||
# Check that the current thread is spawning a child process
|
||||
#
|
||||
|
||||
|
||||
def assert_spawning(self):
|
||||
if not Popen.thread_is_spawning():
|
||||
raise RuntimeError(
|
||||
'%s objects should only be shared between processes'
|
||||
' through inheritance' % type(self).__name__
|
||||
)
|
||||
|
||||
|
||||
#
|
||||
# Unix
|
||||
#
|
||||
|
||||
if sys.platform != 'win32':
|
||||
try:
|
||||
import thread
|
||||
except ImportError:
|
||||
import _thread as thread # noqa
|
||||
import select
|
||||
|
||||
WINEXE = False
|
||||
WINSERVICE = False
|
||||
|
||||
exit = os._exit
|
||||
duplicate = os.dup
|
||||
close = os.close
|
||||
_select = util._eintr_retry(select.select)
|
||||
|
||||
#
|
||||
# We define a Popen class similar to the one from subprocess, but
|
||||
# whose constructor takes a process object as its argument.
|
||||
#
|
||||
|
||||
class Popen(object):
|
||||
|
||||
_tls = thread._local()
|
||||
|
||||
def __init__(self, process_obj):
|
||||
# register reducers
|
||||
from billiard import connection # noqa
|
||||
_Django_old_layout_hack__save()
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
self.returncode = None
|
||||
r, w = os.pipe()
|
||||
self.sentinel = r
|
||||
|
||||
if _forking_is_enabled:
|
||||
self.pid = os.fork()
|
||||
if self.pid == 0:
|
||||
os.close(r)
|
||||
if 'random' in sys.modules:
|
||||
import random
|
||||
random.seed()
|
||||
code = process_obj._bootstrap()
|
||||
os._exit(code)
|
||||
else:
|
||||
from_parent_fd, to_child_fd = os.pipe()
|
||||
cmd = get_command_line() + [str(from_parent_fd)]
|
||||
|
||||
self.pid = os.fork()
|
||||
if self.pid == 0:
|
||||
os.close(r)
|
||||
os.close(to_child_fd)
|
||||
os.execv(sys.executable, cmd)
|
||||
|
||||
# send information to child
|
||||
prep_data = get_preparation_data(process_obj._name)
|
||||
os.close(from_parent_fd)
|
||||
to_child = os.fdopen(to_child_fd, 'wb')
|
||||
Popen._tls.process_handle = self.pid
|
||||
try:
|
||||
dump(prep_data, to_child, HIGHEST_PROTOCOL)
|
||||
dump(process_obj, to_child, HIGHEST_PROTOCOL)
|
||||
finally:
|
||||
del(Popen._tls.process_handle)
|
||||
to_child.close()
|
||||
|
||||
# `w` will be closed when the child exits, at which point `r`
|
||||
# will become ready for reading (using e.g. select()).
|
||||
os.close(w)
|
||||
util.Finalize(self, os.close, (self.sentinel,))
|
||||
|
||||
def poll(self, flag=os.WNOHANG):
|
||||
if self.returncode is None:
|
||||
try:
|
||||
pid, sts = os.waitpid(self.pid, flag)
|
||||
except os.error:
|
||||
# Child process not yet created. See #1731717
|
||||
# e.errno == errno.ECHILD == 10
|
||||
return None
|
||||
if pid == self.pid:
|
||||
if os.WIFSIGNALED(sts):
|
||||
self.returncode = -os.WTERMSIG(sts)
|
||||
else:
|
||||
assert os.WIFEXITED(sts)
|
||||
self.returncode = os.WEXITSTATUS(sts)
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout=None):
|
||||
if self.returncode is None:
|
||||
if timeout is not None:
|
||||
r = _select([self.sentinel], [], [], timeout)[0]
|
||||
if not r:
|
||||
return None
|
||||
# This shouldn't block if select() returned successfully.
|
||||
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
|
||||
return self.returncode
|
||||
|
||||
def terminate(self):
|
||||
if self.returncode is None:
|
||||
try:
|
||||
os.kill(self.pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
if self.wait(timeout=0.1) is None:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def thread_is_spawning():
|
||||
if _forking_is_enabled:
|
||||
return False
|
||||
else:
|
||||
return getattr(Popen._tls, 'process_handle', None) is not None
|
||||
|
||||
@staticmethod
|
||||
def duplicate_for_child(handle):
|
||||
return handle
|
||||
|
||||
#
|
||||
# Windows
|
||||
#
|
||||
|
||||
else:
|
||||
try:
|
||||
import thread
|
||||
except ImportError:
|
||||
import _thread as thread # noqa
|
||||
import msvcrt
|
||||
try:
|
||||
import _subprocess
|
||||
except ImportError:
|
||||
import _winapi as _subprocess # noqa
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
TERMINATE = 0x10000
|
||||
WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
|
||||
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
|
||||
|
||||
exit = win32.ExitProcess
|
||||
close = win32.CloseHandle
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
def duplicate(handle, target_process=None, inheritable=False):
|
||||
if target_process is None:
|
||||
target_process = _subprocess.GetCurrentProcess()
|
||||
h = _subprocess.DuplicateHandle(
|
||||
_subprocess.GetCurrentProcess(), handle, target_process,
|
||||
0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS
|
||||
)
|
||||
if sys.version_info[0] < 3 or (
|
||||
sys.version_info[0] == 3 and sys.version_info[1] < 3):
|
||||
h = h.Detach()
|
||||
return h
|
||||
|
||||
#
|
||||
# We define a Popen class similar to the one from subprocess, but
|
||||
# whose constructor takes a process object as its argument.
|
||||
#
|
||||
|
||||
class Popen(object):
|
||||
'''
|
||||
Start a subprocess to run the code of a process object
|
||||
'''
|
||||
_tls = thread._local()
|
||||
|
||||
def __init__(self, process_obj):
|
||||
_Django_old_layout_hack__save()
|
||||
# create pipe for communication with child
|
||||
rfd, wfd = os.pipe()
|
||||
|
||||
# get handle for read end of the pipe and make it inheritable
|
||||
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
|
||||
os.close(rfd)
|
||||
|
||||
# start process
|
||||
cmd = get_command_line() + [rhandle]
|
||||
cmd = ' '.join('"%s"' % x for x in cmd)
|
||||
hp, ht, pid, tid = _subprocess.CreateProcess(
|
||||
_python_exe, cmd, None, None, 1, 0, None, None, None
|
||||
)
|
||||
close(ht) if isinstance(ht, int_types) else ht.Close()
|
||||
(close(rhandle) if isinstance(rhandle, int_types)
|
||||
else rhandle.Close())
|
||||
|
||||
# set attributes of self
|
||||
self.pid = pid
|
||||
self.returncode = None
|
||||
self._handle = hp
|
||||
self.sentinel = int(hp)
|
||||
|
||||
# send information to child
|
||||
prep_data = get_preparation_data(process_obj._name)
|
||||
to_child = os.fdopen(wfd, 'wb')
|
||||
Popen._tls.process_handle = int(hp)
|
||||
try:
|
||||
dump(prep_data, to_child, HIGHEST_PROTOCOL)
|
||||
dump(process_obj, to_child, HIGHEST_PROTOCOL)
|
||||
finally:
|
||||
del Popen._tls.process_handle
|
||||
to_child.close()
|
||||
|
||||
@staticmethod
|
||||
def thread_is_spawning():
|
||||
return getattr(Popen._tls, 'process_handle', None) is not None
|
||||
|
||||
@staticmethod
|
||||
def duplicate_for_child(handle):
|
||||
return duplicate(handle, Popen._tls.process_handle)
|
||||
|
||||
def wait(self, timeout=None):
|
||||
if self.returncode is None:
|
||||
if timeout is None:
|
||||
msecs = _subprocess.INFINITE
|
||||
else:
|
||||
msecs = max(0, int(timeout * 1000 + 0.5))
|
||||
|
||||
res = _subprocess.WaitForSingleObject(int(self._handle), msecs)
|
||||
if res == _subprocess.WAIT_OBJECT_0:
|
||||
code = _subprocess.GetExitCodeProcess(self._handle)
|
||||
if code == TERMINATE:
|
||||
code = -signal.SIGTERM
|
||||
self.returncode = code
|
||||
|
||||
return self.returncode
|
||||
|
||||
def poll(self):
|
||||
return self.wait(timeout=0)
|
||||
|
||||
def terminate(self):
|
||||
if self.returncode is None:
|
||||
try:
|
||||
_subprocess.TerminateProcess(int(self._handle), TERMINATE)
|
||||
except WindowsError:
|
||||
if self.wait(timeout=0.1) is None:
|
||||
raise
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
if WINSERVICE:
|
||||
_python_exe = os.path.join(sys.exec_prefix, 'python.exe')
|
||||
else:
|
||||
_python_exe = sys.executable
|
||||
|
||||
|
||||
def set_executable(exe):
|
||||
global _python_exe
|
||||
_python_exe = exe
|
||||
|
||||
|
||||
def is_forking(argv):
|
||||
'''
|
||||
Return whether commandline indicates we are forking
|
||||
'''
|
||||
if len(argv) >= 2 and argv[1] == '--billiard-fork':
|
||||
assert len(argv) == 3
|
||||
os.environ["FORKED_BY_MULTIPROCESSING"] = "1"
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def freeze_support():
|
||||
'''
|
||||
Run code for process object if this in not the main process
|
||||
'''
|
||||
if is_forking(sys.argv):
|
||||
main()
|
||||
sys.exit()
|
||||
|
||||
|
||||
def get_command_line():
|
||||
'''
|
||||
Returns prefix of command line used for spawning a child process
|
||||
'''
|
||||
if process.current_process()._identity == () and is_forking(sys.argv):
|
||||
raise RuntimeError('''
|
||||
Attempt to start a new process before the current process
|
||||
has finished its bootstrapping phase.
|
||||
|
||||
This probably means that have forgotten to use the proper
|
||||
idiom in the main module:
|
||||
|
||||
if __name__ == '__main__':
|
||||
freeze_support()
|
||||
...
|
||||
|
||||
The "freeze_support()" line can be omitted if the program
|
||||
is not going to be frozen to produce a Windows executable.''')
|
||||
|
||||
if getattr(sys, 'frozen', False):
|
||||
return [sys.executable, '--billiard-fork']
|
||||
else:
|
||||
prog = 'from billiard.forking import main; main()'
|
||||
return [_python_exe, '-c', prog, '--billiard-fork']
|
||||
|
||||
|
||||
def _Django_old_layout_hack__save():
|
||||
if 'DJANGO_PROJECT_DIR' not in os.environ:
|
||||
try:
|
||||
settings_name = os.environ['DJANGO_SETTINGS_MODULE']
|
||||
except KeyError:
|
||||
return # not using Django.
|
||||
|
||||
conf_settings = sys.modules.get('django.conf.settings')
|
||||
configured = conf_settings and conf_settings.configured
|
||||
try:
|
||||
project_name, _ = settings_name.split('.', 1)
|
||||
except ValueError:
|
||||
return # not modified by setup_environ
|
||||
|
||||
project = __import__(project_name)
|
||||
try:
|
||||
project_dir = os.path.normpath(_module_parent_dir(project))
|
||||
except AttributeError:
|
||||
return # dynamically generated module (no __file__)
|
||||
if configured:
|
||||
warnings.warn(UserWarning(
|
||||
W_OLD_DJANGO_LAYOUT % os.path.realpath(project_dir)
|
||||
))
|
||||
os.environ['DJANGO_PROJECT_DIR'] = project_dir
|
||||
|
||||
|
||||
def _Django_old_layout_hack__load():
|
||||
try:
|
||||
sys.path.append(os.environ['DJANGO_PROJECT_DIR'])
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def _module_parent_dir(mod):
|
||||
dir, filename = os.path.split(_module_dir(mod))
|
||||
if dir == os.curdir or not dir:
|
||||
dir = os.getcwd()
|
||||
return dir
|
||||
|
||||
|
||||
def _module_dir(mod):
|
||||
if '__init__.py' in mod.__file__:
|
||||
return os.path.dirname(mod.__file__)
|
||||
return mod.__file__
|
||||
|
||||
|
||||
def main():
|
||||
'''
|
||||
Run code specifed by data received over pipe
|
||||
'''
|
||||
global _forking_is_enabled
|
||||
_Django_old_layout_hack__load()
|
||||
|
||||
assert is_forking(sys.argv)
|
||||
_forking_is_enabled = False
|
||||
|
||||
handle = int(sys.argv[-1])
|
||||
if sys.platform == 'win32':
|
||||
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
|
||||
else:
|
||||
fd = handle
|
||||
from_parent = os.fdopen(fd, 'rb')
|
||||
|
||||
process.current_process()._inheriting = True
|
||||
preparation_data = load(from_parent)
|
||||
prepare(preparation_data)
|
||||
# Huge hack to make logging before Process.run work.
|
||||
try:
|
||||
os.environ["MP_MAIN_FILE"] = sys.modules["__main__"].__file__
|
||||
except KeyError:
|
||||
pass
|
||||
except AttributeError:
|
||||
pass
|
||||
loglevel = os.environ.get("_MP_FORK_LOGLEVEL_")
|
||||
logfile = os.environ.get("_MP_FORK_LOGFILE_") or None
|
||||
format = os.environ.get("_MP_FORK_LOGFORMAT_")
|
||||
if loglevel:
|
||||
from billiard import util
|
||||
import logging
|
||||
logger = util.get_logger()
|
||||
logger.setLevel(int(loglevel))
|
||||
if not logger.handlers:
|
||||
logger._rudimentary_setup = True
|
||||
logfile = logfile or sys.__stderr__
|
||||
if hasattr(logfile, "write"):
|
||||
handler = logging.StreamHandler(logfile)
|
||||
else:
|
||||
handler = logging.FileHandler(logfile)
|
||||
formatter = logging.Formatter(
|
||||
format or util.DEFAULT_LOGGING_FORMAT,
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
self = load(from_parent)
|
||||
process.current_process()._inheriting = False
|
||||
|
||||
from_parent.close()
|
||||
|
||||
exitcode = self._bootstrap()
|
||||
exit(exitcode)
|
||||
|
||||
|
||||
def get_preparation_data(name):
|
||||
'''
|
||||
Return info about parent needed by child to unpickle process object
|
||||
'''
|
||||
from billiard.util import _logger, _log_to_stderr
|
||||
|
||||
d = dict(
|
||||
name=name,
|
||||
sys_path=sys.path,
|
||||
sys_argv=sys.argv,
|
||||
log_to_stderr=_log_to_stderr,
|
||||
orig_dir=process.ORIGINAL_DIR,
|
||||
authkey=process.current_process().authkey,
|
||||
)
|
||||
|
||||
if _logger is not None:
|
||||
d['log_level'] = _logger.getEffectiveLevel()
|
||||
|
||||
if not WINEXE and not WINSERVICE:
|
||||
main_path = getattr(sys.modules['__main__'], '__file__', None)
|
||||
if not main_path and sys.argv[0] not in ('', '-c'):
|
||||
main_path = sys.argv[0]
|
||||
if main_path is not None:
|
||||
if (not os.path.isabs(main_path) and
|
||||
process.ORIGINAL_DIR is not None):
|
||||
main_path = os.path.join(process.ORIGINAL_DIR, main_path)
|
||||
d['main_path'] = os.path.normpath(main_path)
|
||||
|
||||
return d
|
||||
|
||||
#
|
||||
# Prepare current process
|
||||
#
|
||||
|
||||
old_main_modules = []
|
||||
|
||||
|
||||
def prepare(data):
|
||||
'''
|
||||
Try to get current process ready to unpickle process object
|
||||
'''
|
||||
old_main_modules.append(sys.modules['__main__'])
|
||||
|
||||
if 'name' in data:
|
||||
process.current_process().name = data['name']
|
||||
|
||||
if 'authkey' in data:
|
||||
process.current_process()._authkey = data['authkey']
|
||||
|
||||
if 'log_to_stderr' in data and data['log_to_stderr']:
|
||||
util.log_to_stderr()
|
||||
|
||||
if 'log_level' in data:
|
||||
util.get_logger().setLevel(data['log_level'])
|
||||
|
||||
if 'sys_path' in data:
|
||||
sys.path = data['sys_path']
|
||||
|
||||
if 'sys_argv' in data:
|
||||
sys.argv = data['sys_argv']
|
||||
|
||||
if 'dir' in data:
|
||||
os.chdir(data['dir'])
|
||||
|
||||
if 'orig_dir' in data:
|
||||
process.ORIGINAL_DIR = data['orig_dir']
|
||||
|
||||
if 'main_path' in data:
|
||||
main_path = data['main_path']
|
||||
main_name = os.path.splitext(os.path.basename(main_path))[0]
|
||||
if main_name == '__init__':
|
||||
main_name = os.path.basename(os.path.dirname(main_path))
|
||||
|
||||
if main_name == '__main__':
|
||||
main_module = sys.modules['__main__']
|
||||
main_module.__file__ = main_path
|
||||
elif main_name != 'ipython':
|
||||
# Main modules not actually called __main__.py may
|
||||
# contain additional code that should still be executed
|
||||
import imp
|
||||
|
||||
if main_path is None:
|
||||
dirs = None
|
||||
elif os.path.basename(main_path).startswith('__init__.py'):
|
||||
dirs = [os.path.dirname(os.path.dirname(main_path))]
|
||||
else:
|
||||
dirs = [os.path.dirname(main_path)]
|
||||
|
||||
assert main_name not in sys.modules, main_name
|
||||
file, path_name, etc = imp.find_module(main_name, dirs)
|
||||
try:
|
||||
# We would like to do "imp.load_module('__main__', ...)"
|
||||
# here. However, that would cause 'if __name__ ==
|
||||
# "__main__"' clauses to be executed.
|
||||
main_module = imp.load_module(
|
||||
'__parents_main__', file, path_name, etc
|
||||
)
|
||||
finally:
|
||||
if file:
|
||||
file.close()
|
||||
|
||||
sys.modules['__main__'] = main_module
|
||||
main_module.__name__ = '__main__'
|
||||
|
||||
# Try to make the potentially picklable objects in
|
||||
# sys.modules['__main__'] realize they are in the main
|
||||
# module -- somewhat ugly.
|
||||
for obj in list(main_module.__dict__.values()):
|
||||
try:
|
||||
if obj.__module__ == '__parents_main__':
|
||||
obj.__module__ = '__main__'
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,255 @@
|
||||
#
|
||||
# Module which supports allocation of memory from an mmap
|
||||
#
|
||||
# multiprocessing/heap.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import bisect
|
||||
import mmap
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import itertools
|
||||
|
||||
from ._ext import _billiard, win32
|
||||
from .util import Finalize, info, get_temp_dir
|
||||
from .forking import assert_spawning
|
||||
from .reduction import ForkingPickler
|
||||
|
||||
__all__ = ['BufferWrapper']
|
||||
|
||||
try:
|
||||
maxsize = sys.maxsize
|
||||
except AttributeError:
|
||||
maxsize = sys.maxint
|
||||
|
||||
#
|
||||
# Inheirtable class which wraps an mmap, and from which blocks can be allocated
|
||||
#
|
||||
|
||||
if sys.platform == 'win32':
|
||||
|
||||
class Arena(object):
|
||||
|
||||
_counter = itertools.count()
|
||||
|
||||
def __init__(self, size):
|
||||
self.size = size
|
||||
self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
|
||||
self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
|
||||
assert win32.GetLastError() == 0, 'tagname already in use'
|
||||
self._state = (self.size, self.name)
|
||||
|
||||
def __getstate__(self):
|
||||
assert_spawning(self)
|
||||
return self._state
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.size, self.name = self._state = state
|
||||
self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
|
||||
assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
|
||||
|
||||
else:
|
||||
|
||||
class Arena(object):
|
||||
|
||||
_counter = itertools.count()
|
||||
|
||||
def __init__(self, size, fileno=-1):
|
||||
from .forking import _forking_is_enabled
|
||||
self.size = size
|
||||
self.fileno = fileno
|
||||
if fileno == -1 and not _forking_is_enabled:
|
||||
name = os.path.join(
|
||||
get_temp_dir(),
|
||||
'pym-%d-%d' % (os.getpid(), next(self._counter)))
|
||||
self.fileno = os.open(
|
||||
name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
os.unlink(name)
|
||||
os.ftruncate(self.fileno, size)
|
||||
self.buffer = mmap.mmap(self.fileno, self.size)
|
||||
|
||||
def reduce_arena(a):
|
||||
if a.fileno == -1:
|
||||
raise ValueError('Arena is unpicklable because'
|
||||
'forking was enabled when it was created')
|
||||
return Arena, (a.size, a.fileno)
|
||||
|
||||
ForkingPickler.register(Arena, reduce_arena)
|
||||
|
||||
#
|
||||
# Class allowing allocation of chunks of memory from arenas
|
||||
#
|
||||
|
||||
|
||||
class Heap(object):
|
||||
|
||||
_alignment = 8
|
||||
|
||||
def __init__(self, size=mmap.PAGESIZE):
|
||||
self._lastpid = os.getpid()
|
||||
self._lock = threading.Lock()
|
||||
self._size = size
|
||||
self._lengths = []
|
||||
self._len_to_seq = {}
|
||||
self._start_to_block = {}
|
||||
self._stop_to_block = {}
|
||||
self._allocated_blocks = set()
|
||||
self._arenas = []
|
||||
# list of pending blocks to free - see free() comment below
|
||||
self._pending_free_blocks = []
|
||||
|
||||
@staticmethod
|
||||
def _roundup(n, alignment):
|
||||
# alignment must be a power of 2
|
||||
mask = alignment - 1
|
||||
return (n + mask) & ~mask
|
||||
|
||||
def _malloc(self, size):
|
||||
# returns a large enough block -- it might be much larger
|
||||
i = bisect.bisect_left(self._lengths, size)
|
||||
if i == len(self._lengths):
|
||||
length = self._roundup(max(self._size, size), mmap.PAGESIZE)
|
||||
self._size *= 2
|
||||
info('allocating a new mmap of length %d', length)
|
||||
arena = Arena(length)
|
||||
self._arenas.append(arena)
|
||||
return (arena, 0, length)
|
||||
else:
|
||||
length = self._lengths[i]
|
||||
seq = self._len_to_seq[length]
|
||||
block = seq.pop()
|
||||
if not seq:
|
||||
del self._len_to_seq[length], self._lengths[i]
|
||||
|
||||
(arena, start, stop) = block
|
||||
del self._start_to_block[(arena, start)]
|
||||
del self._stop_to_block[(arena, stop)]
|
||||
return block
|
||||
|
||||
def _free(self, block):
|
||||
# free location and try to merge with neighbours
|
||||
(arena, start, stop) = block
|
||||
|
||||
try:
|
||||
prev_block = self._stop_to_block[(arena, start)]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
start, _ = self._absorb(prev_block)
|
||||
|
||||
try:
|
||||
next_block = self._start_to_block[(arena, stop)]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
_, stop = self._absorb(next_block)
|
||||
|
||||
block = (arena, start, stop)
|
||||
length = stop - start
|
||||
|
||||
try:
|
||||
self._len_to_seq[length].append(block)
|
||||
except KeyError:
|
||||
self._len_to_seq[length] = [block]
|
||||
bisect.insort(self._lengths, length)
|
||||
|
||||
self._start_to_block[(arena, start)] = block
|
||||
self._stop_to_block[(arena, stop)] = block
|
||||
|
||||
def _absorb(self, block):
|
||||
# deregister this block so it can be merged with a neighbour
|
||||
(arena, start, stop) = block
|
||||
del self._start_to_block[(arena, start)]
|
||||
del self._stop_to_block[(arena, stop)]
|
||||
|
||||
length = stop - start
|
||||
seq = self._len_to_seq[length]
|
||||
seq.remove(block)
|
||||
if not seq:
|
||||
del self._len_to_seq[length]
|
||||
self._lengths.remove(length)
|
||||
|
||||
return start, stop
|
||||
|
||||
def _free_pending_blocks(self):
|
||||
# Free all the blocks in the pending list - called with the lock held
|
||||
while 1:
|
||||
try:
|
||||
block = self._pending_free_blocks.pop()
|
||||
except IndexError:
|
||||
break
|
||||
self._allocated_blocks.remove(block)
|
||||
self._free(block)
|
||||
|
||||
def free(self, block):
|
||||
# free a block returned by malloc()
|
||||
# Since free() can be called asynchronously by the GC, it could happen
|
||||
# that it's called while self._lock is held: in that case,
|
||||
# self._lock.acquire() would deadlock (issue #12352). To avoid that, a
|
||||
# trylock is used instead, and if the lock can't be acquired
|
||||
# immediately, the block is added to a list of blocks to be freed
|
||||
# synchronously sometimes later from malloc() or free(), by calling
|
||||
# _free_pending_blocks() (appending and retrieving from a list is not
|
||||
# strictly thread-safe but under cPython it's atomic thanks
|
||||
# to the GIL).
|
||||
assert os.getpid() == self._lastpid
|
||||
if not self._lock.acquire(False):
|
||||
# can't aquire the lock right now, add the block to the list of
|
||||
# pending blocks to free
|
||||
self._pending_free_blocks.append(block)
|
||||
else:
|
||||
# we hold the lock
|
||||
try:
|
||||
self._free_pending_blocks()
|
||||
self._allocated_blocks.remove(block)
|
||||
self._free(block)
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def malloc(self, size):
|
||||
# return a block of right size (possibly rounded up)
|
||||
assert 0 <= size < maxsize
|
||||
if os.getpid() != self._lastpid:
|
||||
self.__init__() # reinitialize after fork
|
||||
self._lock.acquire()
|
||||
self._free_pending_blocks()
|
||||
try:
|
||||
size = self._roundup(max(size, 1), self._alignment)
|
||||
(arena, start, stop) = self._malloc(size)
|
||||
new_stop = start + size
|
||||
if new_stop < stop:
|
||||
self._free((arena, new_stop, stop))
|
||||
block = (arena, start, new_stop)
|
||||
self._allocated_blocks.add(block)
|
||||
return block
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
#
|
||||
# Class representing a chunk of an mmap -- can be inherited
|
||||
#
|
||||
|
||||
|
||||
class BufferWrapper(object):
|
||||
|
||||
_heap = Heap()
|
||||
|
||||
def __init__(self, size):
|
||||
assert 0 <= size < maxsize
|
||||
block = BufferWrapper._heap.malloc(size)
|
||||
self._state = (block, size)
|
||||
Finalize(self, BufferWrapper._heap.free, args=(block,))
|
||||
|
||||
def get_address(self):
|
||||
(arena, start, stop), size = self._state
|
||||
address, length = _billiard.address_of_buffer(arena.buffer)
|
||||
assert size <= length
|
||||
return address + start
|
||||
|
||||
def get_size(self):
|
||||
return self._state[1]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,368 @@
|
||||
#
|
||||
# Module providing the `Process` class which emulates `threading.Thread`
|
||||
#
|
||||
# multiprocessing/process.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
#
|
||||
# Imports
|
||||
#
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import itertools
|
||||
import binascii
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from multiprocessing import process as _mproc
|
||||
|
||||
from .compat import bytes
|
||||
try:
|
||||
from _weakrefset import WeakSet
|
||||
except ImportError:
|
||||
WeakSet = None # noqa
|
||||
from .five import items, string_t
|
||||
|
||||
try:
|
||||
ORIGINAL_DIR = os.path.abspath(os.getcwd())
|
||||
except OSError:
|
||||
ORIGINAL_DIR = None
|
||||
|
||||
__all__ = ['Process', 'current_process', 'active_children']
|
||||
|
||||
#
|
||||
# Public functions
|
||||
#
|
||||
|
||||
|
||||
def current_process():
|
||||
'''
|
||||
Return process object representing the current process
|
||||
'''
|
||||
return _current_process
|
||||
|
||||
|
||||
def _set_current_process(process):
|
||||
global _current_process
|
||||
_current_process = _mproc._current_process = process
|
||||
|
||||
|
||||
def _cleanup():
|
||||
# check for processes which have finished
|
||||
if _current_process is not None:
|
||||
for p in list(_current_process._children):
|
||||
if p._popen.poll() is not None:
|
||||
_current_process._children.discard(p)
|
||||
|
||||
|
||||
def _maybe_flush(f):
|
||||
try:
|
||||
f.flush()
|
||||
except (AttributeError, EnvironmentError, NotImplementedError):
|
||||
pass
|
||||
|
||||
|
||||
def active_children(_cleanup=_cleanup):
|
||||
'''
|
||||
Return list of process objects corresponding to live child processes
|
||||
'''
|
||||
try:
|
||||
_cleanup()
|
||||
except TypeError:
|
||||
# called after gc collect so _cleanup does not exist anymore
|
||||
return []
|
||||
if _current_process is not None:
|
||||
return list(_current_process._children)
|
||||
return []
|
||||
|
||||
|
||||
class Process(object):
|
||||
'''
|
||||
Process objects represent activity that is run in a separate process
|
||||
|
||||
The class is analagous to `threading.Thread`
|
||||
'''
|
||||
_Popen = None
|
||||
|
||||
def __init__(self, group=None, target=None, name=None,
|
||||
args=(), kwargs={}, daemon=None, **_kw):
|
||||
assert group is None, 'group argument must be None for now'
|
||||
count = next(_current_process._counter)
|
||||
self._identity = _current_process._identity + (count,)
|
||||
self._authkey = _current_process._authkey
|
||||
if daemon is not None:
|
||||
self._daemonic = daemon
|
||||
else:
|
||||
self._daemonic = _current_process._daemonic
|
||||
self._tempdir = _current_process._tempdir
|
||||
self._semprefix = _current_process._semprefix
|
||||
self._unlinkfd = _current_process._unlinkfd
|
||||
self._parent_pid = os.getpid()
|
||||
self._popen = None
|
||||
self._target = target
|
||||
self._args = tuple(args)
|
||||
self._kwargs = dict(kwargs)
|
||||
self._name = (
|
||||
name or type(self).__name__ + '-' +
|
||||
':'.join(str(i) for i in self._identity)
|
||||
)
|
||||
if _dangling is not None:
|
||||
_dangling.add(self)
|
||||
|
||||
def run(self):
|
||||
'''
|
||||
Method to be run in sub-process; can be overridden in sub-class
|
||||
'''
|
||||
if self._target:
|
||||
self._target(*self._args, **self._kwargs)
|
||||
|
||||
def start(self):
|
||||
'''
|
||||
Start child process
|
||||
'''
|
||||
assert self._popen is None, 'cannot start a process twice'
|
||||
assert self._parent_pid == os.getpid(), \
|
||||
'can only start a process object created by current process'
|
||||
_cleanup()
|
||||
if self._Popen is not None:
|
||||
Popen = self._Popen
|
||||
else:
|
||||
from .forking import Popen
|
||||
self._popen = Popen(self)
|
||||
self._sentinel = self._popen.sentinel
|
||||
_current_process._children.add(self)
|
||||
|
||||
def terminate(self):
|
||||
'''
|
||||
Terminate process; sends SIGTERM signal or uses TerminateProcess()
|
||||
'''
|
||||
self._popen.terminate()
|
||||
|
||||
def join(self, timeout=None):
|
||||
'''
|
||||
Wait until child process terminates
|
||||
'''
|
||||
assert self._parent_pid == os.getpid(), 'can only join a child process'
|
||||
assert self._popen is not None, 'can only join a started process'
|
||||
res = self._popen.wait(timeout)
|
||||
if res is not None:
|
||||
_current_process._children.discard(self)
|
||||
|
||||
def is_alive(self):
|
||||
'''
|
||||
Return whether process is alive
|
||||
'''
|
||||
if self is _current_process:
|
||||
return True
|
||||
assert self._parent_pid == os.getpid(), 'can only test a child process'
|
||||
if self._popen is None:
|
||||
return False
|
||||
self._popen.poll()
|
||||
return self._popen.returncode is None
|
||||
|
||||
def _is_alive(self):
|
||||
if self._popen is None:
|
||||
return False
|
||||
return self._popen.poll() is None
|
||||
|
||||
def _get_name(self):
|
||||
return self._name
|
||||
|
||||
def _set_name(self, value):
|
||||
assert isinstance(name, string_t), 'name must be a string'
|
||||
self._name = value
|
||||
name = property(_get_name, _set_name)
|
||||
|
||||
def _get_daemon(self):
|
||||
return self._daemonic
|
||||
|
||||
def _set_daemon(self, daemonic):
|
||||
assert self._popen is None, 'process has already started'
|
||||
self._daemonic = daemonic
|
||||
daemon = property(_get_daemon, _set_daemon)
|
||||
|
||||
def _get_authkey(self):
|
||||
return self._authkey
|
||||
|
||||
def _set_authkey(self, authkey):
|
||||
self._authkey = AuthenticationString(authkey)
|
||||
authkey = property(_get_authkey, _set_authkey)
|
||||
|
||||
@property
|
||||
def exitcode(self):
|
||||
'''
|
||||
Return exit code of process or `None` if it has yet to stop
|
||||
'''
|
||||
if self._popen is None:
|
||||
return self._popen
|
||||
return self._popen.poll()
|
||||
|
||||
@property
|
||||
def ident(self):
|
||||
'''
|
||||
Return identifier (PID) of process or `None` if it has yet to start
|
||||
'''
|
||||
if self is _current_process:
|
||||
return os.getpid()
|
||||
else:
|
||||
return self._popen and self._popen.pid
|
||||
|
||||
pid = ident
|
||||
|
||||
@property
|
||||
def sentinel(self):
|
||||
'''
|
||||
Return a file descriptor (Unix) or handle (Windows) suitable for
|
||||
waiting for process termination.
|
||||
'''
|
||||
try:
|
||||
return self._sentinel
|
||||
except AttributeError:
|
||||
raise ValueError("process not started")
|
||||
|
||||
def __repr__(self):
|
||||
if self is _current_process:
|
||||
status = 'started'
|
||||
elif self._parent_pid != os.getpid():
|
||||
status = 'unknown'
|
||||
elif self._popen is None:
|
||||
status = 'initial'
|
||||
else:
|
||||
if self._popen.poll() is not None:
|
||||
status = self.exitcode
|
||||
else:
|
||||
status = 'started'
|
||||
|
||||
if type(status) is int:
|
||||
if status == 0:
|
||||
status = 'stopped'
|
||||
else:
|
||||
status = 'stopped[%s]' % _exitcode_to_name.get(status, status)
|
||||
|
||||
return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,
|
||||
status, self._daemonic and ' daemon' or '')
|
||||
|
||||
##
|
||||
|
||||
def _bootstrap(self):
|
||||
from . import util
|
||||
global _current_process
|
||||
|
||||
try:
|
||||
self._children = set()
|
||||
self._counter = itertools.count(1)
|
||||
if sys.stdin is not None:
|
||||
try:
|
||||
sys.stdin.close()
|
||||
sys.stdin = open(os.devnull)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
old_process = _current_process
|
||||
_set_current_process(self)
|
||||
|
||||
# Re-init logging system.
|
||||
# Workaround for http://bugs.python.org/issue6721/#msg140215
|
||||
# Python logging module uses RLock() objects which are broken
|
||||
# after fork. This can result in a deadlock (Celery Issue #496).
|
||||
loggerDict = logging.Logger.manager.loggerDict
|
||||
logger_names = list(loggerDict.keys())
|
||||
logger_names.append(None) # for root logger
|
||||
for name in logger_names:
|
||||
if not name or not isinstance(loggerDict[name],
|
||||
logging.PlaceHolder):
|
||||
for handler in logging.getLogger(name).handlers:
|
||||
handler.createLock()
|
||||
logging._lock = threading.RLock()
|
||||
|
||||
try:
|
||||
util._finalizer_registry.clear()
|
||||
util._run_after_forkers()
|
||||
finally:
|
||||
# delay finalization of the old process object until after
|
||||
# _run_after_forkers() is executed
|
||||
del old_process
|
||||
util.info('child process %s calling self.run()', self.pid)
|
||||
try:
|
||||
self.run()
|
||||
exitcode = 0
|
||||
finally:
|
||||
util._exit_function()
|
||||
except SystemExit as exc:
|
||||
if not exc.args:
|
||||
exitcode = 1
|
||||
elif isinstance(exc.args[0], int):
|
||||
exitcode = exc.args[0]
|
||||
else:
|
||||
sys.stderr.write(str(exc.args[0]) + '\n')
|
||||
_maybe_flush(sys.stderr)
|
||||
exitcode = 0 if isinstance(exc.args[0], str) else 1
|
||||
except:
|
||||
exitcode = 1
|
||||
if not util.error('Process %s', self.name, exc_info=True):
|
||||
import traceback
|
||||
sys.stderr.write('Process %s:\n' % self.name)
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
util.info('process %s exiting with exitcode %d',
|
||||
self.pid, exitcode)
|
||||
_maybe_flush(sys.stdout)
|
||||
_maybe_flush(sys.stderr)
|
||||
return exitcode
|
||||
|
||||
#
|
||||
# We subclass bytes to avoid accidental transmission of auth keys over network
|
||||
#
|
||||
|
||||
|
||||
class AuthenticationString(bytes):
|
||||
|
||||
def __reduce__(self):
|
||||
from .forking import Popen
|
||||
|
||||
if not Popen.thread_is_spawning():
|
||||
raise TypeError(
|
||||
'Pickling an AuthenticationString object is '
|
||||
'disallowed for security reasons')
|
||||
return AuthenticationString, (bytes(self),)
|
||||
|
||||
#
|
||||
# Create object representing the main process
|
||||
#
|
||||
|
||||
|
||||
class _MainProcess(Process):
|
||||
|
||||
def __init__(self):
|
||||
self._identity = ()
|
||||
self._daemonic = False
|
||||
self._name = 'MainProcess'
|
||||
self._parent_pid = None
|
||||
self._popen = None
|
||||
self._counter = itertools.count(1)
|
||||
self._children = set()
|
||||
self._authkey = AuthenticationString(os.urandom(32))
|
||||
self._tempdir = None
|
||||
self._semprefix = 'mp-' + binascii.hexlify(
|
||||
os.urandom(4)).decode('ascii')
|
||||
self._unlinkfd = None
|
||||
|
||||
_current_process = _MainProcess()
|
||||
del _MainProcess
|
||||
|
||||
#
|
||||
# Give names to some return codes
|
||||
#
|
||||
|
||||
_exitcode_to_name = {}
|
||||
|
||||
for name, signum in items(signal.__dict__):
|
||||
if name[:3] == 'SIG' and '_' not in name:
|
||||
_exitcode_to_name[-signum] = name
|
||||
|
||||
_dangling = WeakSet() if WeakSet is not None else None
|
||||
@@ -0,0 +1,965 @@
|
||||
#
|
||||
# A higher level module for using sockets (or Windows named pipes)
|
||||
#
|
||||
# multiprocessing/connection.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import select
|
||||
import socket
|
||||
import struct
|
||||
import errno
|
||||
import tempfile
|
||||
import itertools
|
||||
|
||||
import _multiprocessing
|
||||
from ..compat import setblocking
|
||||
from ..exceptions import AuthenticationError, BufferTooShort
|
||||
from ..five import monotonic
|
||||
from ..util import get_temp_dir, Finalize, sub_debug
|
||||
from ..reduction import ForkingPickler
|
||||
|
||||
try:
|
||||
import _winapi
|
||||
|
||||
WAIT_OBJECT_0 = _winapi.WAIT_OBJECT_0
|
||||
WAIT_TIMEOUT = _winapi.WAIT_TIMEOUT
|
||||
INFINITE = _winapi.INFINITE
|
||||
# if we got here, we seem to be running on Windows. Handle probably
|
||||
# missing WAIT_ABANDONED_0 constant:
|
||||
try:
|
||||
WAIT_ABANDONED_0 = _winapi.WAIT_ABANDONED_0
|
||||
except ImportError:
|
||||
# _winapi seems to be not exporting
|
||||
# this constant, fallback solution until
|
||||
# exported in _winapio
|
||||
WAIT_ABANDONED_0 = 128
|
||||
except ImportError:
|
||||
if sys.platform == 'win32':
|
||||
raise
|
||||
_winapi = None
|
||||
|
||||
__all__ = ['Client', 'Listener', 'Pipe', 'wait']
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
BUFSIZE = 8192
|
||||
# A very generous timeout when it comes to local connections...
|
||||
CONNECTION_TIMEOUT = 20.
|
||||
|
||||
_mmap_counter = itertools.count()
|
||||
|
||||
default_family = 'AF_INET'
|
||||
families = ['AF_INET']
|
||||
|
||||
if hasattr(socket, 'AF_UNIX'):
|
||||
default_family = 'AF_UNIX'
|
||||
families += ['AF_UNIX']
|
||||
|
||||
if sys.platform == 'win32':
|
||||
default_family = 'AF_PIPE'
|
||||
families += ['AF_PIPE']
|
||||
|
||||
|
||||
def _init_timeout(timeout=CONNECTION_TIMEOUT):
|
||||
return monotonic() + timeout
|
||||
|
||||
|
||||
def _check_timeout(t):
|
||||
return monotonic() > t
|
||||
|
||||
|
||||
def arbitrary_address(family):
|
||||
'''
|
||||
Return an arbitrary free address for the given family
|
||||
'''
|
||||
if family == 'AF_INET':
|
||||
return ('localhost', 0)
|
||||
elif family == 'AF_UNIX':
|
||||
return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
|
||||
elif family == 'AF_PIPE':
|
||||
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
|
||||
(os.getpid(), next(_mmap_counter)))
|
||||
else:
|
||||
raise ValueError('unrecognized family')
|
||||
|
||||
|
||||
def _validate_family(family):
|
||||
'''
|
||||
Checks if the family is valid for the current environment.
|
||||
'''
|
||||
if sys.platform != 'win32' and family == 'AF_PIPE':
|
||||
raise ValueError('Family %s is not recognized.' % family)
|
||||
|
||||
if sys.platform == 'win32' and family == 'AF_UNIX':
|
||||
# double check
|
||||
if not hasattr(socket, family):
|
||||
raise ValueError('Family %s is not recognized.' % family)
|
||||
|
||||
|
||||
def address_type(address):
|
||||
'''
|
||||
Return the types of the address
|
||||
|
||||
This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
|
||||
'''
|
||||
if type(address) == tuple:
|
||||
return 'AF_INET'
|
||||
elif type(address) is str and address.startswith('\\\\'):
|
||||
return 'AF_PIPE'
|
||||
elif type(address) is str:
|
||||
return 'AF_UNIX'
|
||||
else:
|
||||
raise ValueError('address type of %r unrecognized' % address)
|
||||
|
||||
#
|
||||
# Connection classes
|
||||
#
|
||||
|
||||
|
||||
class _ConnectionBase:
|
||||
_handle = None
|
||||
|
||||
def __init__(self, handle, readable=True, writable=True):
|
||||
handle = handle.__index__()
|
||||
if handle < 0:
|
||||
raise ValueError("invalid handle")
|
||||
if not readable and not writable:
|
||||
raise ValueError(
|
||||
"at least one of `readable` and `writable` must be True")
|
||||
self._handle = handle
|
||||
self._readable = readable
|
||||
self._writable = writable
|
||||
|
||||
# XXX should we use util.Finalize instead of a __del__?
|
||||
|
||||
def __del__(self):
|
||||
if self._handle is not None:
|
||||
self._close()
|
||||
|
||||
def _check_closed(self):
|
||||
if self._handle is None:
|
||||
raise OSError("handle is closed")
|
||||
|
||||
def _check_readable(self):
|
||||
if not self._readable:
|
||||
raise OSError("connection is write-only")
|
||||
|
||||
def _check_writable(self):
|
||||
if not self._writable:
|
||||
raise OSError("connection is read-only")
|
||||
|
||||
def _bad_message_length(self):
|
||||
if self._writable:
|
||||
self._readable = False
|
||||
else:
|
||||
self.close()
|
||||
raise OSError("bad message length")
|
||||
|
||||
@property
|
||||
def closed(self):
|
||||
"""True if the connection is closed"""
|
||||
return self._handle is None
|
||||
|
||||
@property
|
||||
def readable(self):
|
||||
"""True if the connection is readable"""
|
||||
return self._readable
|
||||
|
||||
@property
|
||||
def writable(self):
|
||||
"""True if the connection is writable"""
|
||||
return self._writable
|
||||
|
||||
def fileno(self):
|
||||
"""File descriptor or handle of the connection"""
|
||||
self._check_closed()
|
||||
return self._handle
|
||||
|
||||
def close(self):
|
||||
"""Close the connection"""
|
||||
if self._handle is not None:
|
||||
try:
|
||||
self._close()
|
||||
finally:
|
||||
self._handle = None
|
||||
|
||||
def send_bytes(self, buf, offset=0, size=None):
|
||||
"""Send the bytes data from a bytes-like object"""
|
||||
self._check_closed()
|
||||
self._check_writable()
|
||||
m = memoryview(buf)
|
||||
# HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
|
||||
if m.itemsize > 1:
|
||||
m = memoryview(bytes(m))
|
||||
n = len(m)
|
||||
if offset < 0:
|
||||
raise ValueError("offset is negative")
|
||||
if n < offset:
|
||||
raise ValueError("buffer length < offset")
|
||||
if size is None:
|
||||
size = n - offset
|
||||
elif size < 0:
|
||||
raise ValueError("size is negative")
|
||||
elif offset + size > n:
|
||||
raise ValueError("buffer length < offset + size")
|
||||
self._send_bytes(m[offset:offset + size])
|
||||
|
||||
def send(self, obj):
|
||||
"""Send a (picklable) object"""
|
||||
self._check_closed()
|
||||
self._check_writable()
|
||||
self._send_bytes(ForkingPickler.dumps(obj))
|
||||
|
||||
def recv_bytes(self, maxlength=None):
|
||||
"""
|
||||
Receive bytes data as a bytes object.
|
||||
"""
|
||||
self._check_closed()
|
||||
self._check_readable()
|
||||
if maxlength is not None and maxlength < 0:
|
||||
raise ValueError("negative maxlength")
|
||||
buf = self._recv_bytes(maxlength)
|
||||
if buf is None:
|
||||
self._bad_message_length()
|
||||
return buf.getvalue()
|
||||
|
||||
def recv_bytes_into(self, buf, offset=0):
|
||||
"""
|
||||
Receive bytes data into a writeable buffer-like object.
|
||||
Return the number of bytes read.
|
||||
"""
|
||||
self._check_closed()
|
||||
self._check_readable()
|
||||
with memoryview(buf) as m:
|
||||
# Get bytesize of arbitrary buffer
|
||||
itemsize = m.itemsize
|
||||
bytesize = itemsize * len(m)
|
||||
if offset < 0:
|
||||
raise ValueError("negative offset")
|
||||
elif offset > bytesize:
|
||||
raise ValueError("offset too large")
|
||||
result = self._recv_bytes()
|
||||
size = result.tell()
|
||||
if bytesize < offset + size:
|
||||
raise BufferTooShort(result.getvalue())
|
||||
# Message can fit in dest
|
||||
result.seek(0)
|
||||
result.readinto(
|
||||
m[offset // itemsize:(offset + size) // itemsize]
|
||||
)
|
||||
return size
|
||||
|
||||
def recv_payload(self):
|
||||
return self._recv_bytes().getbuffer()
|
||||
|
||||
def recv(self):
|
||||
"""Receive a (picklable) object"""
|
||||
self._check_closed()
|
||||
self._check_readable()
|
||||
buf = self._recv_bytes()
|
||||
return ForkingPickler.loads(buf.getbuffer())
|
||||
|
||||
def poll(self, timeout=0.0):
|
||||
"""Whether there is any input available to be read"""
|
||||
self._check_closed()
|
||||
self._check_readable()
|
||||
return self._poll(timeout)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
self.close()
|
||||
|
||||
|
||||
if _winapi:
|
||||
|
||||
class PipeConnection(_ConnectionBase):
|
||||
"""
|
||||
Connection class based on a Windows named pipe.
|
||||
Overlapped I/O is used, so the handles must have been created
|
||||
with FILE_FLAG_OVERLAPPED.
|
||||
"""
|
||||
_got_empty_message = False
|
||||
|
||||
def _close(self, _CloseHandle=_winapi.CloseHandle):
|
||||
_CloseHandle(self._handle)
|
||||
|
||||
def _send_bytes(self, buf):
|
||||
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
waitres = _winapi.WaitForMultipleObjects(
|
||||
[ov.event], False, INFINITE)
|
||||
assert waitres == WAIT_OBJECT_0
|
||||
except:
|
||||
ov.cancel()
|
||||
raise
|
||||
finally:
|
||||
nwritten, err = ov.GetOverlappedResult(True)
|
||||
assert err == 0
|
||||
assert nwritten == len(buf)
|
||||
|
||||
def _recv_bytes(self, maxsize=None):
|
||||
if self._got_empty_message:
|
||||
self._got_empty_message = False
|
||||
return io.BytesIO()
|
||||
else:
|
||||
bsize = 128 if maxsize is None else min(maxsize, 128)
|
||||
try:
|
||||
ov, err = _winapi.ReadFile(self._handle, bsize,
|
||||
overlapped=True)
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
waitres = _winapi.WaitForMultipleObjects(
|
||||
[ov.event], False, INFINITE)
|
||||
assert waitres == WAIT_OBJECT_0
|
||||
except:
|
||||
ov.cancel()
|
||||
raise
|
||||
finally:
|
||||
nread, err = ov.GetOverlappedResult(True)
|
||||
if err == 0:
|
||||
f = io.BytesIO()
|
||||
f.write(ov.getbuffer())
|
||||
return f
|
||||
elif err == _winapi.ERROR_MORE_DATA:
|
||||
return self._get_more_data(ov, maxsize)
|
||||
except OSError as e:
|
||||
if e.winerror == _winapi.ERROR_BROKEN_PIPE:
|
||||
raise EOFError
|
||||
else:
|
||||
raise
|
||||
raise RuntimeError(
|
||||
"shouldn't get here; expected KeyboardInterrupt"
|
||||
)
|
||||
|
||||
def _poll(self, timeout):
|
||||
if (self._got_empty_message or
|
||||
_winapi.PeekNamedPipe(self._handle)[0] != 0):
|
||||
return True
|
||||
return bool(wait([self], timeout))
|
||||
|
||||
def _get_more_data(self, ov, maxsize):
|
||||
buf = ov.getbuffer()
|
||||
f = io.BytesIO()
|
||||
f.write(buf)
|
||||
left = _winapi.PeekNamedPipe(self._handle)[1]
|
||||
assert left > 0
|
||||
if maxsize is not None and len(buf) + left > maxsize:
|
||||
self._bad_message_length()
|
||||
ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
|
||||
rbytes, err = ov.GetOverlappedResult(True)
|
||||
assert err == 0
|
||||
assert rbytes == left
|
||||
f.write(ov.getbuffer())
|
||||
return f
|
||||
|
||||
|
||||
class Connection(_ConnectionBase):
|
||||
"""
|
||||
Connection class based on an arbitrary file descriptor (Unix only), or
|
||||
a socket handle (Windows).
|
||||
"""
|
||||
|
||||
if _winapi:
|
||||
def _close(self, _close=_multiprocessing.closesocket):
|
||||
_close(self._handle)
|
||||
_write = _multiprocessing.send
|
||||
_read = _multiprocessing.recv
|
||||
else:
|
||||
def _close(self, _close=os.close): # noqa
|
||||
_close(self._handle)
|
||||
_write = os.write
|
||||
_read = os.read
|
||||
|
||||
def send_offset(self, buf, offset, write=_write):
|
||||
return write(self._handle, buf[offset:])
|
||||
|
||||
def _send(self, buf, write=_write):
|
||||
remaining = len(buf)
|
||||
while True:
|
||||
try:
|
||||
n = write(self._handle, buf)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EINTR:
|
||||
continue
|
||||
raise
|
||||
remaining -= n
|
||||
if remaining == 0:
|
||||
break
|
||||
buf = buf[n:]
|
||||
|
||||
def setblocking(self, blocking):
|
||||
setblocking(self._handle, blocking)
|
||||
|
||||
def _recv(self, size, read=_read):
|
||||
buf = io.BytesIO()
|
||||
handle = self._handle
|
||||
remaining = size
|
||||
while remaining > 0:
|
||||
try:
|
||||
chunk = read(handle, remaining)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EINTR:
|
||||
continue
|
||||
raise
|
||||
n = len(chunk)
|
||||
if n == 0:
|
||||
if remaining == size:
|
||||
raise EOFError
|
||||
else:
|
||||
raise OSError("got end of file during message")
|
||||
buf.write(chunk)
|
||||
remaining -= n
|
||||
return buf
|
||||
|
||||
def _send_bytes(self, buf):
|
||||
# For wire compatibility with 3.2 and lower
|
||||
n = len(buf)
|
||||
self._send(struct.pack("!i", n))
|
||||
# The condition is necessary to avoid "broken pipe" errors
|
||||
# when sending a 0-length buffer if the other end closed the pipe.
|
||||
if n > 0:
|
||||
self._send(buf)
|
||||
|
||||
def _recv_bytes(self, maxsize=None):
|
||||
buf = self._recv(4)
|
||||
size, = struct.unpack("!i", buf.getvalue())
|
||||
if maxsize is not None and size > maxsize:
|
||||
return None
|
||||
return self._recv(size)
|
||||
|
||||
def _poll(self, timeout):
|
||||
r = wait([self], timeout)
|
||||
return bool(r)
|
||||
|
||||
|
||||
#
|
||||
# Public functions
|
||||
#
|
||||
|
||||
class Listener(object):
|
||||
'''
|
||||
Returns a listener object.
|
||||
|
||||
This is a wrapper for a bound socket which is 'listening' for
|
||||
connections, or for a Windows named pipe.
|
||||
'''
|
||||
def __init__(self, address=None, family=None, backlog=1, authkey=None):
|
||||
family = (family or (address and address_type(address)) or
|
||||
default_family)
|
||||
address = address or arbitrary_address(family)
|
||||
|
||||
_validate_family(family)
|
||||
if family == 'AF_PIPE':
|
||||
self._listener = PipeListener(address, backlog)
|
||||
else:
|
||||
self._listener = SocketListener(address, family, backlog)
|
||||
|
||||
if authkey is not None and not isinstance(authkey, bytes):
|
||||
raise TypeError('authkey should be a byte string')
|
||||
|
||||
self._authkey = authkey
|
||||
|
||||
def accept(self):
|
||||
'''
|
||||
Accept a connection on the bound socket or named pipe of `self`.
|
||||
|
||||
Returns a `Connection` object.
|
||||
'''
|
||||
if self._listener is None:
|
||||
raise OSError('listener is closed')
|
||||
c = self._listener.accept()
|
||||
if self._authkey:
|
||||
deliver_challenge(c, self._authkey)
|
||||
answer_challenge(c, self._authkey)
|
||||
return c
|
||||
|
||||
def close(self):
|
||||
'''
|
||||
Close the bound socket or named pipe of `self`.
|
||||
'''
|
||||
if self._listener is not None:
|
||||
self._listener.close()
|
||||
self._listener = None
|
||||
|
||||
address = property(lambda self: self._listener._address)
|
||||
last_accepted = property(lambda self: self._listener._last_accepted)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
self.close()
|
||||
|
||||
|
||||
def Client(address, family=None, authkey=None):
|
||||
'''
|
||||
Returns a connection to the address of a `Listener`
|
||||
'''
|
||||
family = family or address_type(address)
|
||||
_validate_family(family)
|
||||
if family == 'AF_PIPE':
|
||||
c = PipeClient(address)
|
||||
else:
|
||||
c = SocketClient(address)
|
||||
|
||||
if authkey is not None and not isinstance(authkey, bytes):
|
||||
raise TypeError('authkey should be a byte string')
|
||||
|
||||
if authkey is not None:
|
||||
answer_challenge(c, authkey)
|
||||
deliver_challenge(c, authkey)
|
||||
|
||||
return c
|
||||
|
||||
|
||||
if sys.platform != 'win32':
|
||||
|
||||
def Pipe(duplex=True, rnonblock=False, wnonblock=False):
|
||||
'''
|
||||
Returns pair of connection objects at either end of a pipe
|
||||
'''
|
||||
if duplex:
|
||||
s1, s2 = socket.socketpair()
|
||||
s1.setblocking(not rnonblock)
|
||||
s2.setblocking(not wnonblock)
|
||||
c1 = Connection(s1.detach())
|
||||
c2 = Connection(s2.detach())
|
||||
else:
|
||||
fd1, fd2 = os.pipe()
|
||||
if rnonblock:
|
||||
setblocking(fd1, 0)
|
||||
if wnonblock:
|
||||
setblocking(fd2, 0)
|
||||
c1 = Connection(fd1, writable=False)
|
||||
c2 = Connection(fd2, readable=False)
|
||||
|
||||
return c1, c2
|
||||
|
||||
else:
|
||||
from billiard.forking import duplicate
|
||||
|
||||
def Pipe(duplex=True, rnonblock=False, wnonblock=False): # noqa
|
||||
'''
|
||||
Returns pair of connection objects at either end of a pipe
|
||||
'''
|
||||
address = arbitrary_address('AF_PIPE')
|
||||
if duplex:
|
||||
openmode = _winapi.PIPE_ACCESS_DUPLEX
|
||||
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
|
||||
obsize, ibsize = BUFSIZE, BUFSIZE
|
||||
else:
|
||||
openmode = _winapi.PIPE_ACCESS_INBOUND
|
||||
access = _winapi.GENERIC_WRITE
|
||||
obsize, ibsize = 0, BUFSIZE
|
||||
|
||||
h1 = _winapi.CreateNamedPipe(
|
||||
address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
|
||||
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
|
||||
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
|
||||
_winapi.PIPE_WAIT,
|
||||
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
|
||||
)
|
||||
h2 = _winapi.CreateFile(
|
||||
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
|
||||
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
|
||||
)
|
||||
_winapi.SetNamedPipeHandleState(
|
||||
h2, _winapi.PIPE_READMODE_MESSAGE, None, None
|
||||
)
|
||||
|
||||
overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
|
||||
_, err = overlapped.GetOverlappedResult(True)
|
||||
assert err == 0
|
||||
|
||||
c1 = PipeConnection(duplicate(h1, inheritable=True), writable=duplex)
|
||||
c2 = PipeConnection(duplicate(h2, inheritable=True), readable=duplex)
|
||||
_winapi.CloseHandle(h1)
|
||||
_winapi.CloseHandle(h2)
|
||||
return c1, c2
|
||||
|
||||
#
|
||||
# Definitions for connections based on sockets
|
||||
#
|
||||
|
||||
|
||||
class SocketListener(object):
|
||||
'''
|
||||
Representation of a socket which is bound to an address and listening
|
||||
'''
|
||||
def __init__(self, address, family, backlog=1):
|
||||
self._socket = socket.socket(getattr(socket, family))
|
||||
try:
|
||||
# SO_REUSEADDR has different semantics on Windows (issue #2550).
|
||||
if os.name == 'posix':
|
||||
self._socket.setsockopt(socket.SOL_SOCKET,
|
||||
socket.SO_REUSEADDR, 1)
|
||||
self._socket.setblocking(True)
|
||||
self._socket.bind(address)
|
||||
self._socket.listen(backlog)
|
||||
self._address = self._socket.getsockname()
|
||||
except OSError:
|
||||
self._socket.close()
|
||||
raise
|
||||
self._family = family
|
||||
self._last_accepted = None
|
||||
|
||||
if family == 'AF_UNIX':
|
||||
self._unlink = Finalize(
|
||||
self, os.unlink, args=(address, ), exitpriority=0
|
||||
)
|
||||
else:
|
||||
self._unlink = None
|
||||
|
||||
def accept(self):
|
||||
while True:
|
||||
try:
|
||||
s, self._last_accepted = self._socket.accept()
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EINTR:
|
||||
continue
|
||||
raise
|
||||
else:
|
||||
break
|
||||
s.setblocking(True)
|
||||
return Connection(s.detach())
|
||||
|
||||
def close(self):
|
||||
self._socket.close()
|
||||
if self._unlink is not None:
|
||||
self._unlink()
|
||||
|
||||
|
||||
def SocketClient(address):
|
||||
'''
|
||||
Return a connection object connected to the socket given by `address`
|
||||
'''
|
||||
family = address_type(address)
|
||||
with socket.socket(getattr(socket, family)) as s:
|
||||
s.setblocking(True)
|
||||
s.connect(address)
|
||||
return Connection(s.detach())
|
||||
|
||||
#
|
||||
# Definitions for connections based on named pipes
|
||||
#
|
||||
|
||||
if sys.platform == 'win32':
|
||||
|
||||
class PipeListener(object):
|
||||
'''
|
||||
Representation of a named pipe
|
||||
'''
|
||||
def __init__(self, address, backlog=None):
|
||||
self._address = address
|
||||
self._handle_queue = [self._new_handle(first=True)]
|
||||
|
||||
self._last_accepted = None
|
||||
sub_debug('listener created with address=%r', self._address)
|
||||
self.close = Finalize(
|
||||
self, PipeListener._finalize_pipe_listener,
|
||||
args=(self._handle_queue, self._address), exitpriority=0
|
||||
)
|
||||
|
||||
def _new_handle(self, first=False):
|
||||
flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
|
||||
if first:
|
||||
flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
|
||||
return _winapi.CreateNamedPipe(
|
||||
self._address, flags,
|
||||
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
|
||||
_winapi.PIPE_WAIT,
|
||||
_winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
|
||||
_winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
|
||||
)
|
||||
|
||||
def accept(self):
|
||||
self._handle_queue.append(self._new_handle())
|
||||
handle = self._handle_queue.pop(0)
|
||||
try:
|
||||
ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
|
||||
except OSError as e:
|
||||
if e.winerror != _winapi.ERROR_NO_DATA:
|
||||
raise
|
||||
# ERROR_NO_DATA can occur if a client has already connected,
|
||||
# written data and then disconnected -- see Issue 14725.
|
||||
else:
|
||||
try:
|
||||
_winapi.WaitForMultipleObjects([ov.event], False, INFINITE)
|
||||
except:
|
||||
ov.cancel()
|
||||
_winapi.CloseHandle(handle)
|
||||
raise
|
||||
finally:
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
assert err == 0
|
||||
return PipeConnection(handle)
|
||||
|
||||
@staticmethod
|
||||
def _finalize_pipe_listener(queue, address):
|
||||
sub_debug('closing listener with address=%r', address)
|
||||
for handle in queue:
|
||||
_winapi.CloseHandle(handle)
|
||||
|
||||
def PipeClient(address,
|
||||
errors=(_winapi.ERROR_SEM_TIMEOUT,
|
||||
_winapi.ERROR_PIPE_BUSY)):
|
||||
'''
|
||||
Return a connection object connected to the pipe given by `address`
|
||||
'''
|
||||
t = _init_timeout()
|
||||
while 1:
|
||||
try:
|
||||
_winapi.WaitNamedPipe(address, 1000)
|
||||
h = _winapi.CreateFile(
|
||||
address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
|
||||
0, _winapi.NULL, _winapi.OPEN_EXISTING,
|
||||
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
|
||||
)
|
||||
except OSError as e:
|
||||
if e.winerror not in errors or _check_timeout(t):
|
||||
raise
|
||||
else:
|
||||
break
|
||||
else:
|
||||
raise
|
||||
|
||||
_winapi.SetNamedPipeHandleState(
|
||||
h, _winapi.PIPE_READMODE_MESSAGE, None, None
|
||||
)
|
||||
return PipeConnection(h)
|
||||
|
||||
#
|
||||
# Authentication stuff
|
||||
#
|
||||
|
||||
MESSAGE_LENGTH = 20
|
||||
|
||||
CHALLENGE = b'#CHALLENGE#'
|
||||
WELCOME = b'#WELCOME#'
|
||||
FAILURE = b'#FAILURE#'
|
||||
|
||||
|
||||
def deliver_challenge(connection, authkey):
|
||||
import hmac
|
||||
assert isinstance(authkey, bytes)
|
||||
message = os.urandom(MESSAGE_LENGTH)
|
||||
connection.send_bytes(CHALLENGE + message)
|
||||
digest = hmac.new(authkey, message).digest()
|
||||
response = connection.recv_bytes(256) # reject large message
|
||||
if response == digest:
|
||||
connection.send_bytes(WELCOME)
|
||||
else:
|
||||
connection.send_bytes(FAILURE)
|
||||
raise AuthenticationError('digest received was wrong')
|
||||
|
||||
|
||||
def answer_challenge(connection, authkey):
|
||||
import hmac
|
||||
assert isinstance(authkey, bytes)
|
||||
message = connection.recv_bytes(256) # reject large message
|
||||
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
|
||||
message = message[len(CHALLENGE):]
|
||||
digest = hmac.new(authkey, message).digest()
|
||||
connection.send_bytes(digest)
|
||||
response = connection.recv_bytes(256) # reject large message
|
||||
if response != WELCOME:
|
||||
raise AuthenticationError('digest sent was rejected')
|
||||
|
||||
#
|
||||
# Support for using xmlrpclib for serialization
|
||||
#
|
||||
|
||||
|
||||
class ConnectionWrapper(object):
|
||||
|
||||
def __init__(self, conn, dumps, loads):
|
||||
self._conn = conn
|
||||
self._dumps = dumps
|
||||
self._loads = loads
|
||||
for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
|
||||
obj = getattr(conn, attr)
|
||||
setattr(self, attr, obj)
|
||||
|
||||
def send(self, obj):
|
||||
s = self._dumps(obj)
|
||||
self._conn.send_bytes(s)
|
||||
|
||||
def recv(self):
|
||||
s = self._conn.recv_bytes()
|
||||
return self._loads(s)
|
||||
|
||||
|
||||
def _xml_dumps(obj):
|
||||
return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') # noqa
|
||||
|
||||
|
||||
def _xml_loads(s):
|
||||
(obj,), method = xmlrpclib.loads(s.decode('utf-8')) # noqa
|
||||
return obj
|
||||
|
||||
|
||||
class XmlListener(Listener):
|
||||
def accept(self):
|
||||
global xmlrpclib
|
||||
import xmlrpc.client as xmlrpclib # noqa
|
||||
obj = Listener.accept(self)
|
||||
return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
|
||||
|
||||
|
||||
def XmlClient(*args, **kwds):
|
||||
global xmlrpclib
|
||||
import xmlrpc.client as xmlrpclib # noqa
|
||||
return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
|
||||
|
||||
#
|
||||
# Wait
|
||||
#
|
||||
|
||||
if sys.platform == 'win32':
|
||||
|
||||
def _exhaustive_wait(handles, timeout):
|
||||
# Return ALL handles which are currently signalled. (Only
|
||||
# returning the first signalled might create starvation issues.)
|
||||
L = list(handles)
|
||||
ready = []
|
||||
while L:
|
||||
res = _winapi.WaitForMultipleObjects(L, False, timeout)
|
||||
if res == WAIT_TIMEOUT:
|
||||
break
|
||||
elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
|
||||
res -= WAIT_OBJECT_0
|
||||
elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
|
||||
res -= WAIT_ABANDONED_0
|
||||
else:
|
||||
raise RuntimeError('Should not get here')
|
||||
ready.append(L[res])
|
||||
L = L[res+1:]
|
||||
timeout = 0
|
||||
return ready
|
||||
|
||||
_ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
|
||||
|
||||
def wait(object_list, timeout=None):
|
||||
'''
|
||||
Wait till an object in object_list is ready/readable.
|
||||
|
||||
Returns list of those objects in object_list which are ready/readable.
|
||||
'''
|
||||
if timeout is None:
|
||||
timeout = INFINITE
|
||||
elif timeout < 0:
|
||||
timeout = 0
|
||||
else:
|
||||
timeout = int(timeout * 1000 + 0.5)
|
||||
|
||||
object_list = list(object_list)
|
||||
waithandle_to_obj = {}
|
||||
ov_list = []
|
||||
ready_objects = set()
|
||||
ready_handles = set()
|
||||
|
||||
try:
|
||||
for o in object_list:
|
||||
try:
|
||||
fileno = getattr(o, 'fileno')
|
||||
except AttributeError:
|
||||
waithandle_to_obj[o.__index__()] = o
|
||||
else:
|
||||
# start an overlapped read of length zero
|
||||
try:
|
||||
ov, err = _winapi.ReadFile(fileno(), 0, True)
|
||||
except OSError as e:
|
||||
err = e.winerror
|
||||
if err not in _ready_errors:
|
||||
raise
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
ov_list.append(ov)
|
||||
waithandle_to_obj[ov.event] = o
|
||||
else:
|
||||
# If o.fileno() is an overlapped pipe handle and
|
||||
# err == 0 then there is a zero length message
|
||||
# in the pipe, but it HAS NOT been consumed.
|
||||
ready_objects.add(o)
|
||||
timeout = 0
|
||||
|
||||
ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
|
||||
finally:
|
||||
# request that overlapped reads stop
|
||||
for ov in ov_list:
|
||||
ov.cancel()
|
||||
|
||||
# wait for all overlapped reads to stop
|
||||
for ov in ov_list:
|
||||
try:
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
except OSError as e:
|
||||
err = e.winerror
|
||||
if err not in _ready_errors:
|
||||
raise
|
||||
if err != _winapi.ERROR_OPERATION_ABORTED:
|
||||
o = waithandle_to_obj[ov.event]
|
||||
ready_objects.add(o)
|
||||
if err == 0:
|
||||
# If o.fileno() is an overlapped pipe handle then
|
||||
# a zero length message HAS been consumed.
|
||||
if hasattr(o, '_got_empty_message'):
|
||||
o._got_empty_message = True
|
||||
|
||||
ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
|
||||
return [oj for oj in object_list if oj in ready_objects]
|
||||
|
||||
else:
|
||||
|
||||
if hasattr(select, 'poll'):
|
||||
def _poll(fds, timeout):
|
||||
if timeout is not None:
|
||||
timeout = int(timeout * 1000) # timeout is in milliseconds
|
||||
fd_map = {}
|
||||
pollster = select.poll()
|
||||
for fd in fds:
|
||||
pollster.register(fd, select.POLLIN)
|
||||
if hasattr(fd, 'fileno'):
|
||||
fd_map[fd.fileno()] = fd
|
||||
else:
|
||||
fd_map[fd] = fd
|
||||
ls = []
|
||||
for fd, event in pollster.poll(timeout):
|
||||
if event & select.POLLNVAL:
|
||||
raise ValueError('invalid file descriptor %i' % fd)
|
||||
ls.append(fd_map[fd])
|
||||
return ls
|
||||
else:
|
||||
def _poll(fds, timeout): # noqa
|
||||
return select.select(fds, [], [], timeout)[0]
|
||||
|
||||
def wait(object_list, timeout=None): # noqa
|
||||
'''
|
||||
Wait till an object in object_list is ready/readable.
|
||||
|
||||
Returns list of those objects in object_list which are ready/readable.
|
||||
'''
|
||||
if timeout is not None:
|
||||
if timeout <= 0:
|
||||
return _poll(object_list, 0)
|
||||
else:
|
||||
deadline = monotonic() + timeout
|
||||
while True:
|
||||
try:
|
||||
return _poll(object_list, timeout)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EINTR:
|
||||
raise
|
||||
if timeout is not None:
|
||||
timeout = deadline - monotonic()
|
||||
@@ -0,0 +1,249 @@
|
||||
#
|
||||
# Module which deals with pickling of objects.
|
||||
#
|
||||
# multiprocessing/reduction.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import copyreg
|
||||
import functools
|
||||
import io
|
||||
import os
|
||||
import pickle
|
||||
import socket
|
||||
import sys
|
||||
|
||||
__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
|
||||
|
||||
|
||||
HAVE_SEND_HANDLE = (sys.platform == 'win32' or
|
||||
(hasattr(socket, 'CMSG_LEN') and
|
||||
hasattr(socket, 'SCM_RIGHTS') and
|
||||
hasattr(socket.socket, 'sendmsg')))
|
||||
|
||||
#
|
||||
# Pickler subclass
|
||||
#
|
||||
|
||||
|
||||
class ForkingPickler(pickle.Pickler):
|
||||
'''Pickler subclass used by multiprocessing.'''
|
||||
_extra_reducers = {}
|
||||
_copyreg_dispatch_table = copyreg.dispatch_table
|
||||
|
||||
def __init__(self, *args):
|
||||
super().__init__(*args)
|
||||
self.dispatch_table = self._copyreg_dispatch_table.copy()
|
||||
self.dispatch_table.update(self._extra_reducers)
|
||||
|
||||
@classmethod
|
||||
def register(cls, type, reduce):
|
||||
'''Register a reduce function for a type.'''
|
||||
cls._extra_reducers[type] = reduce
|
||||
|
||||
@classmethod
|
||||
def dumps(cls, obj, protocol=None):
|
||||
buf = io.BytesIO()
|
||||
cls(buf, protocol).dump(obj)
|
||||
return buf.getbuffer()
|
||||
|
||||
loads = pickle.loads
|
||||
|
||||
register = ForkingPickler.register
|
||||
|
||||
|
||||
def dump(obj, file, protocol=None):
|
||||
'''Replacement for pickle.dump() using ForkingPickler.'''
|
||||
ForkingPickler(file, protocol).dump(obj)
|
||||
|
||||
#
|
||||
# Platform specific definitions
|
||||
#
|
||||
|
||||
if sys.platform == 'win32':
|
||||
# Windows
|
||||
__all__ += ['DupHandle', 'duplicate', 'steal_handle']
|
||||
import _winapi
|
||||
|
||||
def duplicate(handle, target_process=None, inheritable=False):
|
||||
'''Duplicate a handle. (target_process is a handle not a pid!)'''
|
||||
if target_process is None:
|
||||
target_process = _winapi.GetCurrentProcess()
|
||||
return _winapi.DuplicateHandle(
|
||||
_winapi.GetCurrentProcess(), handle, target_process,
|
||||
0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
|
||||
|
||||
def steal_handle(source_pid, handle):
|
||||
'''Steal a handle from process identified by source_pid.'''
|
||||
source_process_handle = _winapi.OpenProcess(
|
||||
_winapi.PROCESS_DUP_HANDLE, False, source_pid)
|
||||
try:
|
||||
return _winapi.DuplicateHandle(
|
||||
source_process_handle, handle,
|
||||
_winapi.GetCurrentProcess(), 0, False,
|
||||
_winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
|
||||
finally:
|
||||
_winapi.CloseHandle(source_process_handle)
|
||||
|
||||
def send_handle(conn, handle, destination_pid):
|
||||
'''Send a handle over a local connection.'''
|
||||
dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
|
||||
conn.send(dh)
|
||||
|
||||
def recv_handle(conn):
|
||||
'''Receive a handle over a local connection.'''
|
||||
return conn.recv().detach()
|
||||
|
||||
class DupHandle(object):
|
||||
'''Picklable wrapper for a handle.'''
|
||||
def __init__(self, handle, access, pid=None):
|
||||
if pid is None:
|
||||
# We just duplicate the handle in the current process and
|
||||
# let the receiving process steal the handle.
|
||||
pid = os.getpid()
|
||||
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
|
||||
try:
|
||||
self._handle = _winapi.DuplicateHandle(
|
||||
_winapi.GetCurrentProcess(),
|
||||
handle, proc, access, False, 0)
|
||||
finally:
|
||||
_winapi.CloseHandle(proc)
|
||||
self._access = access
|
||||
self._pid = pid
|
||||
|
||||
def detach(self):
|
||||
'''Get the handle. This should only be called once.'''
|
||||
# retrieve handle from process which currently owns it
|
||||
if self._pid == os.getpid():
|
||||
# The handle has already been duplicated for this process.
|
||||
return self._handle
|
||||
# We must steal the handle from the process whose pid is self._pid.
|
||||
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
|
||||
self._pid)
|
||||
try:
|
||||
return _winapi.DuplicateHandle(
|
||||
proc, self._handle, _winapi.GetCurrentProcess(),
|
||||
self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
|
||||
finally:
|
||||
_winapi.CloseHandle(proc)
|
||||
|
||||
else:
|
||||
# Unix
|
||||
__all__ += ['DupFd', 'sendfds', 'recvfds']
|
||||
import array
|
||||
|
||||
# On MacOSX we should acknowledge receipt of fds -- see Issue14669
|
||||
ACKNOWLEDGE = sys.platform == 'darwin'
|
||||
|
||||
def sendfds(sock, fds):
|
||||
'''Send an array of fds over an AF_UNIX socket.'''
|
||||
fds = array.array('i', fds)
|
||||
msg = bytes([len(fds) % 256])
|
||||
sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
|
||||
if ACKNOWLEDGE and sock.recv(1) != b'A':
|
||||
raise RuntimeError('did not receive acknowledgement of fd')
|
||||
|
||||
def recvfds(sock, size):
|
||||
'''Receive an array of fds over an AF_UNIX socket.'''
|
||||
a = array.array('i')
|
||||
bytes_size = a.itemsize * size
|
||||
msg, ancdata, flags, addr = sock.recvmsg(
|
||||
1, socket.CMSG_LEN(bytes_size),
|
||||
)
|
||||
if not msg and not ancdata:
|
||||
raise EOFError
|
||||
try:
|
||||
if ACKNOWLEDGE:
|
||||
sock.send(b'A')
|
||||
if len(ancdata) != 1:
|
||||
raise RuntimeError(
|
||||
'received %d items of ancdata' % len(ancdata),
|
||||
)
|
||||
cmsg_level, cmsg_type, cmsg_data = ancdata[0]
|
||||
if (cmsg_level == socket.SOL_SOCKET and
|
||||
cmsg_type == socket.SCM_RIGHTS):
|
||||
if len(cmsg_data) % a.itemsize != 0:
|
||||
raise ValueError
|
||||
a.frombytes(cmsg_data)
|
||||
assert len(a) % 256 == msg[0]
|
||||
return list(a)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
raise RuntimeError('Invalid data received')
|
||||
|
||||
def send_handle(conn, handle, destination_pid): # noqa
|
||||
'''Send a handle over a local connection.'''
|
||||
fd = conn.fileno()
|
||||
with socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) as s:
|
||||
sendfds(s, [handle])
|
||||
|
||||
def recv_handle(conn): # noqa
|
||||
'''Receive a handle over a local connection.'''
|
||||
fd = conn.fileno()
|
||||
with socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) as s:
|
||||
return recvfds(s, 1)[0]
|
||||
|
||||
def DupFd(fd):
|
||||
'''Return a wrapper for an fd.'''
|
||||
from ..forking import Popen
|
||||
return Popen.duplicate_for_child(fd)
|
||||
|
||||
#
|
||||
# Try making some callable types picklable
|
||||
#
|
||||
|
||||
|
||||
def _reduce_method(m):
|
||||
if m.__self__ is None:
|
||||
return getattr, (m.__class__, m.__func__.__name__)
|
||||
else:
|
||||
return getattr, (m.__self__, m.__func__.__name__)
|
||||
|
||||
|
||||
class _C:
|
||||
def f(self):
|
||||
pass
|
||||
register(type(_C().f), _reduce_method)
|
||||
|
||||
|
||||
def _reduce_method_descriptor(m):
|
||||
return getattr, (m.__objclass__, m.__name__)
|
||||
register(type(list.append), _reduce_method_descriptor)
|
||||
register(type(int.__add__), _reduce_method_descriptor)
|
||||
|
||||
|
||||
def _reduce_partial(p):
|
||||
return _rebuild_partial, (p.func, p.args, p.keywords or {})
|
||||
|
||||
|
||||
def _rebuild_partial(func, args, keywords):
|
||||
return functools.partial(func, *args, **keywords)
|
||||
register(functools.partial, _reduce_partial)
|
||||
|
||||
#
|
||||
# Make sockets picklable
|
||||
#
|
||||
|
||||
if sys.platform == 'win32':
|
||||
|
||||
def _reduce_socket(s):
|
||||
from ..resource_sharer import DupSocket
|
||||
return _rebuild_socket, (DupSocket(s),)
|
||||
|
||||
def _rebuild_socket(ds):
|
||||
return ds.detach()
|
||||
register(socket.socket, _reduce_socket)
|
||||
|
||||
else:
|
||||
|
||||
def _reduce_socket(s): # noqa
|
||||
df = DupFd(s.fileno())
|
||||
return _rebuild_socket, (df, s.family, s.type, s.proto)
|
||||
|
||||
def _rebuild_socket(df, family, type, proto): # noqa
|
||||
fd = df.detach()
|
||||
return socket.socket(family, type, proto, fileno=fd)
|
||||
register(socket.socket, _reduce_socket)
|
||||
@@ -0,0 +1,372 @@
|
||||
#
|
||||
# Module implementing queues
|
||||
#
|
||||
# multiprocessing/queues.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
import os
|
||||
import threading
|
||||
import collections
|
||||
import weakref
|
||||
import errno
|
||||
|
||||
from . import Pipe
|
||||
from ._ext import _billiard
|
||||
from .compat import get_errno
|
||||
from .five import monotonic
|
||||
from .synchronize import Lock, BoundedSemaphore, Semaphore, Condition
|
||||
from .util import debug, error, info, Finalize, register_after_fork
|
||||
from .five import Empty, Full
|
||||
from .forking import assert_spawning
|
||||
|
||||
__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
|
||||
|
||||
|
||||
class Queue(object):
|
||||
'''
|
||||
Queue type using a pipe, buffer and thread
|
||||
'''
|
||||
def __init__(self, maxsize=0):
|
||||
if maxsize <= 0:
|
||||
maxsize = _billiard.SemLock.SEM_VALUE_MAX
|
||||
self._maxsize = maxsize
|
||||
self._reader, self._writer = Pipe(duplex=False)
|
||||
self._rlock = Lock()
|
||||
self._opid = os.getpid()
|
||||
if sys.platform == 'win32':
|
||||
self._wlock = None
|
||||
else:
|
||||
self._wlock = Lock()
|
||||
self._sem = BoundedSemaphore(maxsize)
|
||||
# For use by concurrent.futures
|
||||
self._ignore_epipe = False
|
||||
|
||||
self._after_fork()
|
||||
|
||||
if sys.platform != 'win32':
|
||||
register_after_fork(self, Queue._after_fork)
|
||||
|
||||
def __getstate__(self):
|
||||
assert_spawning(self)
|
||||
return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
|
||||
self._rlock, self._wlock, self._sem, self._opid)
|
||||
|
||||
def __setstate__(self, state):
|
||||
(self._ignore_epipe, self._maxsize, self._reader, self._writer,
|
||||
self._rlock, self._wlock, self._sem, self._opid) = state
|
||||
self._after_fork()
|
||||
|
||||
def _after_fork(self):
|
||||
debug('Queue._after_fork()')
|
||||
self._notempty = threading.Condition(threading.Lock())
|
||||
self._buffer = collections.deque()
|
||||
self._thread = None
|
||||
self._jointhread = None
|
||||
self._joincancelled = False
|
||||
self._closed = False
|
||||
self._close = None
|
||||
self._send = self._writer.send
|
||||
self._recv = self._reader.recv
|
||||
self._poll = self._reader.poll
|
||||
|
||||
def put(self, obj, block=True, timeout=None):
|
||||
assert not self._closed
|
||||
if not self._sem.acquire(block, timeout):
|
||||
raise Full
|
||||
|
||||
with self._notempty:
|
||||
if self._thread is None:
|
||||
self._start_thread()
|
||||
self._buffer.append(obj)
|
||||
self._notempty.notify()
|
||||
|
||||
def get(self, block=True, timeout=None):
|
||||
if block and timeout is None:
|
||||
with self._rlock:
|
||||
res = self._recv()
|
||||
self._sem.release()
|
||||
return res
|
||||
|
||||
else:
|
||||
if block:
|
||||
deadline = monotonic() + timeout
|
||||
if not self._rlock.acquire(block, timeout):
|
||||
raise Empty
|
||||
try:
|
||||
if block:
|
||||
timeout = deadline - monotonic()
|
||||
if timeout < 0 or not self._poll(timeout):
|
||||
raise Empty
|
||||
elif not self._poll():
|
||||
raise Empty
|
||||
res = self._recv()
|
||||
self._sem.release()
|
||||
return res
|
||||
finally:
|
||||
self._rlock.release()
|
||||
|
||||
def qsize(self):
|
||||
# Raises NotImplementedError on Mac OSX because
|
||||
# of broken sem_getvalue()
|
||||
return self._maxsize - self._sem._semlock._get_value()
|
||||
|
||||
def empty(self):
|
||||
return not self._poll()
|
||||
|
||||
def full(self):
|
||||
return self._sem._semlock._is_zero()
|
||||
|
||||
def get_nowait(self):
|
||||
return self.get(False)
|
||||
|
||||
def put_nowait(self, obj):
|
||||
return self.put(obj, False)
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
self._reader.close()
|
||||
if self._close:
|
||||
self._close()
|
||||
|
||||
def join_thread(self):
|
||||
debug('Queue.join_thread()')
|
||||
assert self._closed
|
||||
if self._jointhread:
|
||||
self._jointhread()
|
||||
|
||||
def cancel_join_thread(self):
|
||||
debug('Queue.cancel_join_thread()')
|
||||
self._joincancelled = True
|
||||
try:
|
||||
self._jointhread.cancel()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
def _start_thread(self):
|
||||
debug('Queue._start_thread()')
|
||||
|
||||
# Start thread which transfers data from buffer to pipe
|
||||
self._buffer.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=Queue._feed,
|
||||
args=(self._buffer, self._notempty, self._send,
|
||||
self._wlock, self._writer.close, self._ignore_epipe),
|
||||
name='QueueFeederThread'
|
||||
)
|
||||
self._thread.daemon = True
|
||||
|
||||
debug('doing self._thread.start()')
|
||||
self._thread.start()
|
||||
debug('... done self._thread.start()')
|
||||
|
||||
# On process exit we will wait for data to be flushed to pipe.
|
||||
#
|
||||
# However, if this process created the queue then all
|
||||
# processes which use the queue will be descendants of this
|
||||
# process. Therefore waiting for the queue to be flushed
|
||||
# is pointless once all the child processes have been joined.
|
||||
created_by_this_process = (self._opid == os.getpid())
|
||||
if not self._joincancelled and not created_by_this_process:
|
||||
self._jointhread = Finalize(
|
||||
self._thread, Queue._finalize_join,
|
||||
[weakref.ref(self._thread)],
|
||||
exitpriority=-5
|
||||
)
|
||||
|
||||
# Send sentinel to the thread queue object when garbage collected
|
||||
self._close = Finalize(
|
||||
self, Queue._finalize_close,
|
||||
[self._buffer, self._notempty],
|
||||
exitpriority=10
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _finalize_join(twr):
|
||||
debug('joining queue thread')
|
||||
thread = twr()
|
||||
if thread is not None:
|
||||
thread.join()
|
||||
debug('... queue thread joined')
|
||||
else:
|
||||
debug('... queue thread already dead')
|
||||
|
||||
@staticmethod
|
||||
def _finalize_close(buffer, notempty):
|
||||
debug('telling queue thread to quit')
|
||||
with notempty:
|
||||
buffer.append(_sentinel)
|
||||
notempty.notify()
|
||||
|
||||
@staticmethod
|
||||
def _feed(buffer, notempty, send, writelock, close, ignore_epipe):
|
||||
debug('starting thread to feed data to pipe')
|
||||
from .util import is_exiting
|
||||
|
||||
ncond = notempty
|
||||
nwait = notempty.wait
|
||||
bpopleft = buffer.popleft
|
||||
sentinel = _sentinel
|
||||
if sys.platform != 'win32':
|
||||
wlock = writelock
|
||||
else:
|
||||
wlock = None
|
||||
|
||||
try:
|
||||
while 1:
|
||||
with ncond:
|
||||
if not buffer:
|
||||
nwait()
|
||||
try:
|
||||
while 1:
|
||||
obj = bpopleft()
|
||||
if obj is sentinel:
|
||||
debug('feeder thread got sentinel -- exiting')
|
||||
close()
|
||||
return
|
||||
|
||||
if wlock is None:
|
||||
send(obj)
|
||||
else:
|
||||
with wlock:
|
||||
send(obj)
|
||||
except IndexError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if ignore_epipe and get_errno(exc) == errno.EPIPE:
|
||||
return
|
||||
# Since this runs in a daemon thread the resources it uses
|
||||
# may be become unusable while the process is cleaning up.
|
||||
# We ignore errors which happen after the process has
|
||||
# started to cleanup.
|
||||
try:
|
||||
if is_exiting():
|
||||
info('error in queue thread: %r', exc, exc_info=True)
|
||||
else:
|
||||
if not error('error in queue thread: %r', exc,
|
||||
exc_info=True):
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_sentinel = object()
|
||||
|
||||
|
||||
class JoinableQueue(Queue):
|
||||
'''
|
||||
A queue type which also supports join() and task_done() methods
|
||||
|
||||
Note that if you do not call task_done() for each finished task then
|
||||
eventually the counter's semaphore may overflow causing Bad Things
|
||||
to happen.
|
||||
'''
|
||||
|
||||
def __init__(self, maxsize=0):
|
||||
Queue.__init__(self, maxsize)
|
||||
self._unfinished_tasks = Semaphore(0)
|
||||
self._cond = Condition()
|
||||
|
||||
def __getstate__(self):
|
||||
return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
|
||||
|
||||
def __setstate__(self, state):
|
||||
Queue.__setstate__(self, state[:-2])
|
||||
self._cond, self._unfinished_tasks = state[-2:]
|
||||
|
||||
def put(self, obj, block=True, timeout=None):
|
||||
assert not self._closed
|
||||
if not self._sem.acquire(block, timeout):
|
||||
raise Full
|
||||
|
||||
with self._notempty:
|
||||
with self._cond:
|
||||
if self._thread is None:
|
||||
self._start_thread()
|
||||
self._buffer.append(obj)
|
||||
self._unfinished_tasks.release()
|
||||
self._notempty.notify()
|
||||
|
||||
def task_done(self):
|
||||
with self._cond:
|
||||
if not self._unfinished_tasks.acquire(False):
|
||||
raise ValueError('task_done() called too many times')
|
||||
if self._unfinished_tasks._semlock._is_zero():
|
||||
self._cond.notify_all()
|
||||
|
||||
def join(self):
|
||||
with self._cond:
|
||||
if not self._unfinished_tasks._semlock._is_zero():
|
||||
self._cond.wait()
|
||||
|
||||
|
||||
class _SimpleQueue(object):
|
||||
'''
|
||||
Simplified Queue type -- really just a locked pipe
|
||||
'''
|
||||
|
||||
def __init__(self, rnonblock=False, wnonblock=False):
|
||||
self._reader, self._writer = Pipe(
|
||||
duplex=False, rnonblock=rnonblock, wnonblock=wnonblock,
|
||||
)
|
||||
self._poll = self._reader.poll
|
||||
self._rlock = self._wlock = None
|
||||
self._make_methods()
|
||||
|
||||
def empty(self):
|
||||
return not self._poll()
|
||||
|
||||
def __getstate__(self):
|
||||
assert_spawning(self)
|
||||
return (self._reader, self._writer, self._rlock, self._wlock)
|
||||
|
||||
def __setstate__(self, state):
|
||||
(self._reader, self._writer, self._rlock, self._wlock) = state
|
||||
self._make_methods()
|
||||
|
||||
def _make_methods(self):
|
||||
recv = self._reader.recv
|
||||
try:
|
||||
recv_payload = self._reader.recv_payload
|
||||
except AttributeError:
|
||||
recv_payload = self._reader.recv_bytes
|
||||
rlock = self._rlock
|
||||
|
||||
if rlock is not None:
|
||||
def get():
|
||||
with rlock:
|
||||
return recv()
|
||||
self.get = get
|
||||
|
||||
def get_payload():
|
||||
with rlock:
|
||||
return recv_payload()
|
||||
self.get_payload = get_payload
|
||||
else:
|
||||
self.get = recv
|
||||
self.get_payload = recv_payload
|
||||
|
||||
if self._wlock is None:
|
||||
# writes to a message oriented win32 pipe are atomic
|
||||
self.put = self._writer.send
|
||||
else:
|
||||
send = self._writer.send
|
||||
wlock = self._wlock
|
||||
|
||||
def put(obj):
|
||||
with wlock:
|
||||
return send(obj)
|
||||
self.put = put
|
||||
|
||||
|
||||
class SimpleQueue(_SimpleQueue):
|
||||
|
||||
def __init__(self):
|
||||
self._reader, self._writer = Pipe(duplex=False)
|
||||
self._rlock = Lock()
|
||||
self._wlock = Lock() if sys.platform != 'win32' else None
|
||||
self._make_methods()
|
||||
@@ -0,0 +1,10 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info[0] == 3:
|
||||
from .py3 import reduction
|
||||
else:
|
||||
from .py2 import reduction # noqa
|
||||
|
||||
sys.modules[__name__] = reduction
|
||||
@@ -0,0 +1,248 @@
|
||||
#
|
||||
# Module which supports allocation of ctypes objects from shared memory
|
||||
#
|
||||
# multiprocessing/sharedctypes.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import ctypes
|
||||
import weakref
|
||||
|
||||
from . import heap, RLock
|
||||
from .five import int_types
|
||||
from .forking import assert_spawning
|
||||
from .reduction import ForkingPickler
|
||||
|
||||
__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
|
||||
|
||||
typecode_to_type = {
|
||||
'c': ctypes.c_char, 'u': ctypes.c_wchar,
|
||||
'b': ctypes.c_byte, 'B': ctypes.c_ubyte,
|
||||
'h': ctypes.c_short, 'H': ctypes.c_ushort,
|
||||
'i': ctypes.c_int, 'I': ctypes.c_uint,
|
||||
'l': ctypes.c_long, 'L': ctypes.c_ulong,
|
||||
'f': ctypes.c_float, 'd': ctypes.c_double
|
||||
}
|
||||
|
||||
|
||||
def _new_value(type_):
|
||||
size = ctypes.sizeof(type_)
|
||||
wrapper = heap.BufferWrapper(size)
|
||||
return rebuild_ctype(type_, wrapper, None)
|
||||
|
||||
|
||||
def RawValue(typecode_or_type, *args):
|
||||
'''
|
||||
Returns a ctypes object allocated from shared memory
|
||||
'''
|
||||
type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
|
||||
obj = _new_value(type_)
|
||||
ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
|
||||
obj.__init__(*args)
|
||||
return obj
|
||||
|
||||
|
||||
def RawArray(typecode_or_type, size_or_initializer):
|
||||
'''
|
||||
Returns a ctypes array allocated from shared memory
|
||||
'''
|
||||
type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
|
||||
if isinstance(size_or_initializer, int_types):
|
||||
type_ = type_ * size_or_initializer
|
||||
obj = _new_value(type_)
|
||||
ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
|
||||
return obj
|
||||
else:
|
||||
type_ = type_ * len(size_or_initializer)
|
||||
result = _new_value(type_)
|
||||
result.__init__(*size_or_initializer)
|
||||
return result
|
||||
|
||||
|
||||
def Value(typecode_or_type, *args, **kwds):
|
||||
'''
|
||||
Return a synchronization wrapper for a Value
|
||||
'''
|
||||
lock = kwds.pop('lock', None)
|
||||
if kwds:
|
||||
raise ValueError(
|
||||
'unrecognized keyword argument(s): %s' % list(kwds.keys()))
|
||||
obj = RawValue(typecode_or_type, *args)
|
||||
if lock is False:
|
||||
return obj
|
||||
if lock in (True, None):
|
||||
lock = RLock()
|
||||
if not hasattr(lock, 'acquire'):
|
||||
raise AttributeError("'%r' has no method 'acquire'" % lock)
|
||||
return synchronized(obj, lock)
|
||||
|
||||
|
||||
def Array(typecode_or_type, size_or_initializer, **kwds):
|
||||
'''
|
||||
Return a synchronization wrapper for a RawArray
|
||||
'''
|
||||
lock = kwds.pop('lock', None)
|
||||
if kwds:
|
||||
raise ValueError(
|
||||
'unrecognized keyword argument(s): %s' % list(kwds.keys()))
|
||||
obj = RawArray(typecode_or_type, size_or_initializer)
|
||||
if lock is False:
|
||||
return obj
|
||||
if lock in (True, None):
|
||||
lock = RLock()
|
||||
if not hasattr(lock, 'acquire'):
|
||||
raise AttributeError("'%r' has no method 'acquire'" % lock)
|
||||
return synchronized(obj, lock)
|
||||
|
||||
|
||||
def copy(obj):
|
||||
new_obj = _new_value(type(obj))
|
||||
ctypes.pointer(new_obj)[0] = obj
|
||||
return new_obj
|
||||
|
||||
|
||||
def synchronized(obj, lock=None):
|
||||
assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
|
||||
|
||||
if isinstance(obj, ctypes._SimpleCData):
|
||||
return Synchronized(obj, lock)
|
||||
elif isinstance(obj, ctypes.Array):
|
||||
if obj._type_ is ctypes.c_char:
|
||||
return SynchronizedString(obj, lock)
|
||||
return SynchronizedArray(obj, lock)
|
||||
else:
|
||||
cls = type(obj)
|
||||
try:
|
||||
scls = class_cache[cls]
|
||||
except KeyError:
|
||||
names = [field[0] for field in cls._fields_]
|
||||
d = dict((name, make_property(name)) for name in names)
|
||||
classname = 'Synchronized' + cls.__name__
|
||||
scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
|
||||
return scls(obj, lock)
|
||||
|
||||
#
|
||||
# Functions for pickling/unpickling
|
||||
#
|
||||
|
||||
|
||||
def reduce_ctype(obj):
|
||||
assert_spawning(obj)
|
||||
if isinstance(obj, ctypes.Array):
|
||||
return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
|
||||
else:
|
||||
return rebuild_ctype, (type(obj), obj._wrapper, None)
|
||||
|
||||
|
||||
def rebuild_ctype(type_, wrapper, length):
|
||||
if length is not None:
|
||||
type_ = type_ * length
|
||||
ForkingPickler.register(type_, reduce_ctype)
|
||||
obj = type_.from_address(wrapper.get_address())
|
||||
obj._wrapper = wrapper
|
||||
return obj
|
||||
|
||||
#
|
||||
# Function to create properties
|
||||
#
|
||||
|
||||
|
||||
def make_property(name):
|
||||
try:
|
||||
return prop_cache[name]
|
||||
except KeyError:
|
||||
d = {}
|
||||
exec(template % ((name, ) * 7), d)
|
||||
prop_cache[name] = d[name]
|
||||
return d[name]
|
||||
|
||||
template = '''
|
||||
def get%s(self):
|
||||
self.acquire()
|
||||
try:
|
||||
return self._obj.%s
|
||||
finally:
|
||||
self.release()
|
||||
def set%s(self, value):
|
||||
self.acquire()
|
||||
try:
|
||||
self._obj.%s = value
|
||||
finally:
|
||||
self.release()
|
||||
%s = property(get%s, set%s)
|
||||
'''
|
||||
|
||||
prop_cache = {}
|
||||
class_cache = weakref.WeakKeyDictionary()
|
||||
|
||||
#
|
||||
# Synchronized wrappers
|
||||
#
|
||||
|
||||
|
||||
class SynchronizedBase(object):
|
||||
|
||||
def __init__(self, obj, lock=None):
|
||||
self._obj = obj
|
||||
self._lock = lock or RLock()
|
||||
self.acquire = self._lock.acquire
|
||||
self.release = self._lock.release
|
||||
|
||||
def __reduce__(self):
|
||||
assert_spawning(self)
|
||||
return synchronized, (self._obj, self._lock)
|
||||
|
||||
def get_obj(self):
|
||||
return self._obj
|
||||
|
||||
def get_lock(self):
|
||||
return self._lock
|
||||
|
||||
def __repr__(self):
|
||||
return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
|
||||
|
||||
|
||||
class Synchronized(SynchronizedBase):
|
||||
value = make_property('value')
|
||||
|
||||
|
||||
class SynchronizedArray(SynchronizedBase):
|
||||
|
||||
def __len__(self):
|
||||
return len(self._obj)
|
||||
|
||||
def __getitem__(self, i):
|
||||
self.acquire()
|
||||
try:
|
||||
return self._obj[i]
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
def __setitem__(self, i, value):
|
||||
self.acquire()
|
||||
try:
|
||||
self._obj[i] = value
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
def __getslice__(self, start, stop):
|
||||
self.acquire()
|
||||
try:
|
||||
return self._obj[start:stop]
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
def __setslice__(self, start, stop, values):
|
||||
self.acquire()
|
||||
try:
|
||||
self._obj[start:stop] = values
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
|
||||
class SynchronizedString(SynchronizedArray):
|
||||
value = make_property('value')
|
||||
raw = make_property('raw')
|
||||
@@ -0,0 +1,449 @@
|
||||
#
|
||||
# Module implementing synchronization primitives
|
||||
#
|
||||
# multiprocessing/synchronize.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
|
||||
|
||||
from ._ext import _billiard, ensure_SemLock
|
||||
from .five import range, monotonic
|
||||
from .process import current_process
|
||||
from .util import Finalize, register_after_fork, debug
|
||||
from .forking import assert_spawning, Popen
|
||||
from .compat import bytes, closerange
|
||||
|
||||
__all__ = [
|
||||
'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event',
|
||||
]
|
||||
|
||||
# Try to import the mp.synchronize module cleanly, if it fails
|
||||
# raise ImportError for platforms lacking a working sem_open implementation.
|
||||
# See issue 3770
|
||||
ensure_SemLock()
|
||||
|
||||
#
|
||||
# Constants
|
||||
#
|
||||
|
||||
RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
|
||||
SEM_VALUE_MAX = _billiard.SemLock.SEM_VALUE_MAX
|
||||
|
||||
try:
|
||||
sem_unlink = _billiard.SemLock.sem_unlink
|
||||
except AttributeError: # pragma: no cover
|
||||
try:
|
||||
# Py3.4+ implements sem_unlink and the semaphore must be named
|
||||
from _multiprocessing import sem_unlink # noqa
|
||||
except ImportError:
|
||||
sem_unlink = None # noqa
|
||||
|
||||
#
|
||||
# Base class for semaphores and mutexes; wraps `_billiard.SemLock`
|
||||
#
|
||||
|
||||
|
||||
def _semname(sl):
|
||||
try:
|
||||
return sl.name
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class SemLock(object):
|
||||
_counter = itertools.count()
|
||||
|
||||
def __init__(self, kind, value, maxvalue):
|
||||
from .forking import _forking_is_enabled
|
||||
unlink_immediately = _forking_is_enabled or sys.platform == 'win32'
|
||||
if sem_unlink:
|
||||
sl = self._semlock = _billiard.SemLock(
|
||||
kind, value, maxvalue, self._make_name(), unlink_immediately)
|
||||
else:
|
||||
sl = self._semlock = _billiard.SemLock(kind, value, maxvalue)
|
||||
|
||||
debug('created semlock with handle %s', sl.handle)
|
||||
self._make_methods()
|
||||
|
||||
if sem_unlink:
|
||||
|
||||
if sys.platform != 'win32':
|
||||
def _after_fork(obj):
|
||||
obj._semlock._after_fork()
|
||||
register_after_fork(self, _after_fork)
|
||||
|
||||
if _semname(self._semlock) is not None:
|
||||
# We only get here if we are on Unix with forking
|
||||
# disabled. When the object is garbage collected or the
|
||||
# process shuts down we unlink the semaphore name
|
||||
Finalize(self, sem_unlink, (self._semlock.name,),
|
||||
exitpriority=0)
|
||||
# In case of abnormal termination unlink semaphore name
|
||||
_cleanup_semaphore_if_leaked(self._semlock.name)
|
||||
|
||||
def _make_methods(self):
|
||||
self.acquire = self._semlock.acquire
|
||||
self.release = self._semlock.release
|
||||
|
||||
def __enter__(self):
|
||||
return self._semlock.__enter__()
|
||||
|
||||
def __exit__(self, *args):
|
||||
return self._semlock.__exit__(*args)
|
||||
|
||||
def __getstate__(self):
|
||||
assert_spawning(self)
|
||||
sl = self._semlock
|
||||
state = (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)
|
||||
try:
|
||||
state += (sl.name, )
|
||||
except AttributeError:
|
||||
pass
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
self._semlock = _billiard.SemLock._rebuild(*state)
|
||||
debug('recreated blocker with handle %r', state[0])
|
||||
self._make_methods()
|
||||
|
||||
@staticmethod
|
||||
def _make_name():
|
||||
return '/%s-%s-%s' % (current_process()._semprefix,
|
||||
os.getpid(), next(SemLock._counter))
|
||||
|
||||
|
||||
class Semaphore(SemLock):
|
||||
|
||||
def __init__(self, value=1):
|
||||
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
|
||||
|
||||
def get_value(self):
|
||||
return self._semlock._get_value()
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
value = self._semlock._get_value()
|
||||
except Exception:
|
||||
value = 'unknown'
|
||||
return '<Semaphore(value=%s)>' % value
|
||||
|
||||
|
||||
class BoundedSemaphore(Semaphore):
|
||||
|
||||
def __init__(self, value=1):
|
||||
SemLock.__init__(self, SEMAPHORE, value, value)
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
value = self._semlock._get_value()
|
||||
except Exception:
|
||||
value = 'unknown'
|
||||
return '<BoundedSemaphore(value=%s, maxvalue=%s)>' % \
|
||||
(value, self._semlock.maxvalue)
|
||||
|
||||
|
||||
class Lock(SemLock):
|
||||
'''
|
||||
Non-recursive lock.
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
SemLock.__init__(self, SEMAPHORE, 1, 1)
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
if self._semlock._is_mine():
|
||||
name = current_process().name
|
||||
if threading.currentThread().name != 'MainThread':
|
||||
name += '|' + threading.currentThread().name
|
||||
elif self._semlock._get_value() == 1:
|
||||
name = 'None'
|
||||
elif self._semlock._count() > 0:
|
||||
name = 'SomeOtherThread'
|
||||
else:
|
||||
name = 'SomeOtherProcess'
|
||||
except Exception:
|
||||
name = 'unknown'
|
||||
return '<Lock(owner=%s)>' % name
|
||||
|
||||
|
||||
class RLock(SemLock):
|
||||
'''
|
||||
Recursive lock
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
if self._semlock._is_mine():
|
||||
name = current_process().name
|
||||
if threading.currentThread().name != 'MainThread':
|
||||
name += '|' + threading.currentThread().name
|
||||
count = self._semlock._count()
|
||||
elif self._semlock._get_value() == 1:
|
||||
name, count = 'None', 0
|
||||
elif self._semlock._count() > 0:
|
||||
name, count = 'SomeOtherThread', 'nonzero'
|
||||
else:
|
||||
name, count = 'SomeOtherProcess', 'nonzero'
|
||||
except Exception:
|
||||
name, count = 'unknown', 'unknown'
|
||||
return '<RLock(%s, %s)>' % (name, count)
|
||||
|
||||
|
||||
class Condition(object):
|
||||
'''
|
||||
Condition variable
|
||||
'''
|
||||
|
||||
def __init__(self, lock=None):
|
||||
self._lock = lock or RLock()
|
||||
self._sleeping_count = Semaphore(0)
|
||||
self._woken_count = Semaphore(0)
|
||||
self._wait_semaphore = Semaphore(0)
|
||||
self._make_methods()
|
||||
|
||||
def __getstate__(self):
|
||||
assert_spawning(self)
|
||||
return (self._lock, self._sleeping_count,
|
||||
self._woken_count, self._wait_semaphore)
|
||||
|
||||
def __setstate__(self, state):
|
||||
(self._lock, self._sleeping_count,
|
||||
self._woken_count, self._wait_semaphore) = state
|
||||
self._make_methods()
|
||||
|
||||
def __enter__(self):
|
||||
return self._lock.__enter__()
|
||||
|
||||
def __exit__(self, *args):
|
||||
return self._lock.__exit__(*args)
|
||||
|
||||
def _make_methods(self):
|
||||
self.acquire = self._lock.acquire
|
||||
self.release = self._lock.release
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
num_waiters = (self._sleeping_count._semlock._get_value() -
|
||||
self._woken_count._semlock._get_value())
|
||||
except Exception:
|
||||
num_waiters = 'unkown'
|
||||
return '<Condition(%s, %s)>' % (self._lock, num_waiters)
|
||||
|
||||
def wait(self, timeout=None):
|
||||
assert self._lock._semlock._is_mine(), \
|
||||
'must acquire() condition before using wait()'
|
||||
|
||||
# indicate that this thread is going to sleep
|
||||
self._sleeping_count.release()
|
||||
|
||||
# release lock
|
||||
count = self._lock._semlock._count()
|
||||
for i in range(count):
|
||||
self._lock.release()
|
||||
|
||||
try:
|
||||
# wait for notification or timeout
|
||||
ret = self._wait_semaphore.acquire(True, timeout)
|
||||
finally:
|
||||
# indicate that this thread has woken
|
||||
self._woken_count.release()
|
||||
|
||||
# reacquire lock
|
||||
for i in range(count):
|
||||
self._lock.acquire()
|
||||
return ret
|
||||
|
||||
def notify(self):
|
||||
assert self._lock._semlock._is_mine(), 'lock is not owned'
|
||||
assert not self._wait_semaphore.acquire(False)
|
||||
|
||||
# to take account of timeouts since last notify() we subtract
|
||||
# woken_count from sleeping_count and rezero woken_count
|
||||
while self._woken_count.acquire(False):
|
||||
res = self._sleeping_count.acquire(False)
|
||||
assert res
|
||||
|
||||
if self._sleeping_count.acquire(False): # try grabbing a sleeper
|
||||
self._wait_semaphore.release() # wake up one sleeper
|
||||
self._woken_count.acquire() # wait for sleeper to wake
|
||||
|
||||
# rezero _wait_semaphore in case a timeout just happened
|
||||
self._wait_semaphore.acquire(False)
|
||||
|
||||
def notify_all(self):
|
||||
assert self._lock._semlock._is_mine(), 'lock is not owned'
|
||||
assert not self._wait_semaphore.acquire(False)
|
||||
|
||||
# to take account of timeouts since last notify*() we subtract
|
||||
# woken_count from sleeping_count and rezero woken_count
|
||||
while self._woken_count.acquire(False):
|
||||
res = self._sleeping_count.acquire(False)
|
||||
assert res
|
||||
|
||||
sleepers = 0
|
||||
while self._sleeping_count.acquire(False):
|
||||
self._wait_semaphore.release() # wake up one sleeper
|
||||
sleepers += 1
|
||||
|
||||
if sleepers:
|
||||
for i in range(sleepers):
|
||||
self._woken_count.acquire() # wait for a sleeper to wake
|
||||
|
||||
# rezero wait_semaphore in case some timeouts just happened
|
||||
while self._wait_semaphore.acquire(False):
|
||||
pass
|
||||
|
||||
def wait_for(self, predicate, timeout=None):
|
||||
result = predicate()
|
||||
if result:
|
||||
return result
|
||||
if timeout is not None:
|
||||
endtime = monotonic() + timeout
|
||||
else:
|
||||
endtime = None
|
||||
waittime = None
|
||||
while not result:
|
||||
if endtime is not None:
|
||||
waittime = endtime - monotonic()
|
||||
if waittime <= 0:
|
||||
break
|
||||
self.wait(waittime)
|
||||
result = predicate()
|
||||
return result
|
||||
|
||||
|
||||
class Event(object):
|
||||
|
||||
def __init__(self):
|
||||
self._cond = Condition(Lock())
|
||||
self._flag = Semaphore(0)
|
||||
|
||||
def is_set(self):
|
||||
self._cond.acquire()
|
||||
try:
|
||||
if self._flag.acquire(False):
|
||||
self._flag.release()
|
||||
return True
|
||||
return False
|
||||
finally:
|
||||
self._cond.release()
|
||||
|
||||
def set(self):
|
||||
self._cond.acquire()
|
||||
try:
|
||||
self._flag.acquire(False)
|
||||
self._flag.release()
|
||||
self._cond.notify_all()
|
||||
finally:
|
||||
self._cond.release()
|
||||
|
||||
def clear(self):
|
||||
self._cond.acquire()
|
||||
try:
|
||||
self._flag.acquire(False)
|
||||
finally:
|
||||
self._cond.release()
|
||||
|
||||
def wait(self, timeout=None):
|
||||
self._cond.acquire()
|
||||
try:
|
||||
if self._flag.acquire(False):
|
||||
self._flag.release()
|
||||
else:
|
||||
self._cond.wait(timeout)
|
||||
|
||||
if self._flag.acquire(False):
|
||||
self._flag.release()
|
||||
return True
|
||||
return False
|
||||
finally:
|
||||
self._cond.release()
|
||||
|
||||
|
||||
if sys.platform != 'win32':
|
||||
#
|
||||
# Protection against unlinked semaphores if the program ends abnormally
|
||||
# and forking has been disabled.
|
||||
#
|
||||
|
||||
def _cleanup_semaphore_if_leaked(name):
|
||||
name = name.encode('ascii') + bytes('\0', 'ascii')
|
||||
if len(name) > 512:
|
||||
# posix guarantees that writes to a pipe of less than PIPE_BUF
|
||||
# bytes are atomic, and that PIPE_BUF >= 512
|
||||
raise ValueError('name too long')
|
||||
fd = _get_unlinkfd()
|
||||
bits = os.write(fd, name)
|
||||
assert bits == len(name)
|
||||
|
||||
def _get_unlinkfd():
|
||||
cp = current_process()
|
||||
if cp._unlinkfd is None:
|
||||
r, w = os.pipe()
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
from setproctitle import setproctitle
|
||||
setproctitle("[sem_cleanup for %r]" % cp.pid)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Fork a process which will survive until all other processes
|
||||
# which have a copy of the write end of the pipe have exited.
|
||||
# The forked process just collects names of semaphores until
|
||||
# EOF is indicated. Then it tries unlinking all the names it
|
||||
# has collected.
|
||||
_collect_names_then_unlink(r)
|
||||
os._exit(0)
|
||||
os.close(r)
|
||||
cp._unlinkfd = w
|
||||
return cp._unlinkfd
|
||||
|
||||
def _collect_names_then_unlink(r):
|
||||
# protect the process from ^C and "killall python" etc
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
|
||||
# close all fds except r
|
||||
try:
|
||||
MAXFD = os.sysconf("SC_OPEN_MAX")
|
||||
except:
|
||||
MAXFD = 256
|
||||
closerange(0, r)
|
||||
closerange(r + 1, MAXFD)
|
||||
|
||||
# collect data written to pipe
|
||||
data = []
|
||||
while 1:
|
||||
try:
|
||||
s = os.read(r, 512)
|
||||
except:
|
||||
# XXX IO lock might be held at fork, so don't try
|
||||
# printing unexpected exception - see issue 6721
|
||||
pass
|
||||
else:
|
||||
if not s:
|
||||
break
|
||||
data.append(s)
|
||||
|
||||
# attempt to unlink each collected name
|
||||
for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')):
|
||||
try:
|
||||
sem_unlink(name.decode('ascii'))
|
||||
except:
|
||||
# XXX IO lock might be held at fork, so don't try
|
||||
# printing unexpected exception - see issue 6721
|
||||
pass
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import atexit
|
||||
|
||||
|
||||
def teardown():
|
||||
# Workaround for multiprocessing bug where logging
|
||||
# is attempted after global already collected at shutdown.
|
||||
cancelled = set()
|
||||
try:
|
||||
import multiprocessing.util
|
||||
cancelled.add(multiprocessing.util._exit_function)
|
||||
except (AttributeError, ImportError):
|
||||
pass
|
||||
|
||||
try:
|
||||
atexit._exithandlers[:] = [
|
||||
e for e in atexit._exithandlers if e[0] not in cancelled
|
||||
]
|
||||
except AttributeError:
|
||||
pass
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
class WarningMessage(object):
|
||||
|
||||
"""Holds the result of a single showwarning() call."""
|
||||
|
||||
_WARNING_DETAILS = ('message', 'category', 'filename', 'lineno', 'file',
|
||||
'line')
|
||||
|
||||
def __init__(self, message, category, filename, lineno, file=None,
|
||||
line=None):
|
||||
local_values = locals()
|
||||
for attr in self._WARNING_DETAILS:
|
||||
setattr(self, attr, local_values[attr])
|
||||
|
||||
self._category_name = category and category.__name__ or None
|
||||
|
||||
def __str__(self):
|
||||
return ('{message : %r, category : %r, filename : %r, lineno : %s, '
|
||||
'line : %r}' % (self.message, self._category_name,
|
||||
self.filename, self.lineno, self.line))
|
||||
|
||||
|
||||
class catch_warnings(object):
|
||||
|
||||
"""A context manager that copies and restores the warnings filter upon
|
||||
exiting the context.
|
||||
|
||||
The 'record' argument specifies whether warnings should be captured by a
|
||||
custom implementation of warnings.showwarning() and be appended to a list
|
||||
returned by the context manager. Otherwise None is returned by the context
|
||||
manager. The objects appended to the list are arguments whose attributes
|
||||
mirror the arguments to showwarning().
|
||||
|
||||
The 'module' argument is to specify an alternative module to the module
|
||||
named 'warnings' and imported under that name. This argument is only
|
||||
useful when testing the warnings module itself.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, record=False, module=None):
|
||||
"""Specify whether to record warnings and if an alternative module
|
||||
should be used other than sys.modules['warnings'].
|
||||
|
||||
For compatibility with Python 3.0, please consider all arguments to be
|
||||
keyword-only.
|
||||
|
||||
"""
|
||||
self._record = record
|
||||
self._module = module is None and sys.modules['warnings'] or module
|
||||
self._entered = False
|
||||
|
||||
def __repr__(self):
|
||||
args = []
|
||||
if self._record:
|
||||
args.append('record=True')
|
||||
if self._module is not sys.modules['warnings']:
|
||||
args.append('module=%r' % self._module)
|
||||
name = type(self).__name__
|
||||
return '%s(%s)' % (name, ', '.join(args))
|
||||
|
||||
def __enter__(self):
|
||||
if self._entered:
|
||||
raise RuntimeError('Cannot enter %r twice' % self)
|
||||
self._entered = True
|
||||
self._filters = self._module.filters
|
||||
self._module.filters = self._filters[:]
|
||||
self._showwarning = self._module.showwarning
|
||||
if self._record:
|
||||
log = []
|
||||
|
||||
def showwarning(*args, **kwargs):
|
||||
log.append(WarningMessage(*args, **kwargs))
|
||||
|
||||
self._module.showwarning = showwarning
|
||||
return log
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
if not self._entered:
|
||||
raise RuntimeError('Cannot exit %r without entering first' % self)
|
||||
self._module.filters = self._filters
|
||||
self._module.showwarning = self._showwarning
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from contextlib import contextmanager
|
||||
from time import time
|
||||
|
||||
from nose import SkipTest
|
||||
from billiard.common import (
|
||||
_shutdown_cleanup,
|
||||
reset_signals,
|
||||
restart_state,
|
||||
)
|
||||
|
||||
from .utils import Case
|
||||
|
||||
try:
|
||||
from unittest.mock import Mock, call, patch
|
||||
except ImportError:
|
||||
from mock import Mock, call, patch # noqa
|
||||
|
||||
|
||||
def signo(name):
|
||||
return getattr(signal, name)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def termsigs(default, full):
|
||||
from billiard import common
|
||||
prev_def, common.TERMSIGS_DEFAULT = common.TERMSIGS_DEFAULT, default
|
||||
prev_full, common.TERMSIGS_FULL = common.TERMSIGS_FULL, full
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
common.TERMSIGS_DEFAULT, common.TERMSIGS_FULL = prev_def, prev_full
|
||||
|
||||
|
||||
class test_reset_signals(Case):
|
||||
|
||||
def setUp(self):
|
||||
if sys.platform == 'win32':
|
||||
raise SkipTest('win32: skip')
|
||||
|
||||
def test_shutdown_handler(self):
|
||||
with patch('sys.exit') as exit:
|
||||
_shutdown_cleanup(15, Mock())
|
||||
self.assertTrue(exit.called)
|
||||
self.assertEqual(os.WTERMSIG(exit.call_args[0][0]), 15)
|
||||
|
||||
def test_does_not_reset_ignored_signal(self, sigs=['SIGTERM']):
|
||||
with self.assert_context(sigs, [], signal.SIG_IGN) as (_, SET):
|
||||
self.assertFalse(SET.called)
|
||||
|
||||
def test_does_not_reset_if_current_is_None(self, sigs=['SIGTERM']):
|
||||
with self.assert_context(sigs, [], None) as (_, SET):
|
||||
self.assertFalse(SET.called)
|
||||
|
||||
def test_resets_for_SIG_DFL(self, sigs=['SIGTERM', 'SIGINT', 'SIGUSR1']):
|
||||
with self.assert_context(sigs, [], signal.SIG_DFL) as (_, SET):
|
||||
SET.assert_has_calls([
|
||||
call(signo(sig), _shutdown_cleanup) for sig in sigs
|
||||
])
|
||||
|
||||
def test_resets_for_obj(self, sigs=['SIGTERM', 'SIGINT', 'SIGUSR1']):
|
||||
with self.assert_context(sigs, [], object()) as (_, SET):
|
||||
SET.assert_has_calls([
|
||||
call(signo(sig), _shutdown_cleanup) for sig in sigs
|
||||
])
|
||||
|
||||
def test_handles_errors(self, sigs=['SIGTERM']):
|
||||
for exc in (OSError(), AttributeError(),
|
||||
ValueError(), RuntimeError()):
|
||||
with self.assert_context(sigs, [], signal.SIG_DFL, exc) as (_, S):
|
||||
self.assertTrue(S.called)
|
||||
|
||||
@contextmanager
|
||||
def assert_context(self, default, full, get_returns=None, set_effect=None):
|
||||
with termsigs(default, full):
|
||||
with patch('signal.getsignal') as GET:
|
||||
with patch('signal.signal') as SET:
|
||||
GET.return_value = get_returns
|
||||
SET.side_effect = set_effect
|
||||
reset_signals()
|
||||
GET.assert_has_calls([
|
||||
call(signo(sig)) for sig in default
|
||||
])
|
||||
yield GET, SET
|
||||
|
||||
|
||||
class test_restart_state(Case):
|
||||
|
||||
def test_raises(self):
|
||||
s = restart_state(100, 1) # max 100 restarts in 1 second.
|
||||
s.R = 99
|
||||
s.step()
|
||||
with self.assertRaises(s.RestartFreqExceeded):
|
||||
s.step()
|
||||
|
||||
def test_time_passed_resets_counter(self):
|
||||
s = restart_state(100, 10)
|
||||
s.R, s.T = 100, time()
|
||||
with self.assertRaises(s.RestartFreqExceeded):
|
||||
s.step()
|
||||
s.R, s.T = 100, time()
|
||||
s.step(time() + 20)
|
||||
self.assertEqual(s.R, 1)
|
||||
@@ -0,0 +1,12 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import billiard
|
||||
|
||||
from .utils import Case
|
||||
|
||||
|
||||
class test_billiard(Case):
|
||||
|
||||
def test_has_version(self):
|
||||
self.assertTrue(billiard.__version__)
|
||||
self.assertIsInstance(billiard.__version__, str)
|
||||
@@ -0,0 +1,145 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
try:
|
||||
import unittest # noqa
|
||||
unittest.skip
|
||||
from unittest.util import safe_repr, unorderable_list_difference
|
||||
except AttributeError:
|
||||
import unittest2 as unittest # noqa
|
||||
from unittest2.util import safe_repr, unorderable_list_difference # noqa
|
||||
|
||||
from billiard.five import string_t, items, values
|
||||
|
||||
from .compat import catch_warnings
|
||||
|
||||
# -- adds assertWarns from recent unittest2, not in Python 2.7.
|
||||
|
||||
|
||||
class _AssertRaisesBaseContext(object):
|
||||
|
||||
def __init__(self, expected, test_case, callable_obj=None,
|
||||
expected_regex=None):
|
||||
self.expected = expected
|
||||
self.failureException = test_case.failureException
|
||||
self.obj_name = None
|
||||
if isinstance(expected_regex, string_t):
|
||||
expected_regex = re.compile(expected_regex)
|
||||
self.expected_regex = expected_regex
|
||||
|
||||
|
||||
class _AssertWarnsContext(_AssertRaisesBaseContext):
|
||||
"""A context manager used to implement TestCase.assertWarns* methods."""
|
||||
|
||||
def __enter__(self):
|
||||
# The __warningregistry__'s need to be in a pristine state for tests
|
||||
# to work properly.
|
||||
warnings.resetwarnings()
|
||||
for v in values(sys.modules):
|
||||
if getattr(v, '__warningregistry__', None):
|
||||
v.__warningregistry__ = {}
|
||||
self.warnings_manager = catch_warnings(record=True)
|
||||
self.warnings = self.warnings_manager.__enter__()
|
||||
warnings.simplefilter('always', self.expected)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, tb):
|
||||
self.warnings_manager.__exit__(exc_type, exc_value, tb)
|
||||
if exc_type is not None:
|
||||
# let unexpected exceptions pass through
|
||||
return
|
||||
try:
|
||||
exc_name = self.expected.__name__
|
||||
except AttributeError:
|
||||
exc_name = str(self.expected)
|
||||
first_matching = None
|
||||
for m in self.warnings:
|
||||
w = m.message
|
||||
if not isinstance(w, self.expected):
|
||||
continue
|
||||
if first_matching is None:
|
||||
first_matching = w
|
||||
if (self.expected_regex is not None and
|
||||
not self.expected_regex.search(str(w))):
|
||||
continue
|
||||
# store warning for later retrieval
|
||||
self.warning = w
|
||||
self.filename = m.filename
|
||||
self.lineno = m.lineno
|
||||
return
|
||||
# Now we simply try to choose a helpful failure message
|
||||
if first_matching is not None:
|
||||
raise self.failureException(
|
||||
'%r does not match %r' % (
|
||||
self.expected_regex.pattern, str(first_matching)))
|
||||
if self.obj_name:
|
||||
raise self.failureException(
|
||||
'%s not triggered by %s' % (exc_name, self.obj_name))
|
||||
else:
|
||||
raise self.failureException('%s not triggered' % exc_name)
|
||||
|
||||
|
||||
class Case(unittest.TestCase):
|
||||
|
||||
def assertWarns(self, expected_warning):
|
||||
return _AssertWarnsContext(expected_warning, self, None)
|
||||
|
||||
def assertWarnsRegex(self, expected_warning, expected_regex):
|
||||
return _AssertWarnsContext(expected_warning, self,
|
||||
None, expected_regex)
|
||||
|
||||
def assertDictContainsSubset(self, expected, actual, msg=None):
|
||||
missing, mismatched = [], []
|
||||
|
||||
for key, value in items(expected):
|
||||
if key not in actual:
|
||||
missing.append(key)
|
||||
elif value != actual[key]:
|
||||
mismatched.append('%s, expected: %s, actual: %s' % (
|
||||
safe_repr(key), safe_repr(value),
|
||||
safe_repr(actual[key])))
|
||||
|
||||
if not (missing or mismatched):
|
||||
return
|
||||
|
||||
standard_msg = ''
|
||||
if missing:
|
||||
standard_msg = 'Missing: %s' % ','.join(map(safe_repr, missing))
|
||||
|
||||
if mismatched:
|
||||
if standard_msg:
|
||||
standard_msg += '; '
|
||||
standard_msg += 'Mismatched values: %s' % (
|
||||
','.join(mismatched))
|
||||
|
||||
self.fail(self._formatMessage(msg, standard_msg))
|
||||
|
||||
def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
|
||||
missing = unexpected = None
|
||||
try:
|
||||
expected = sorted(expected_seq)
|
||||
actual = sorted(actual_seq)
|
||||
except TypeError:
|
||||
# Unsortable items (example: set(), complex(), ...)
|
||||
expected = list(expected_seq)
|
||||
actual = list(actual_seq)
|
||||
missing, unexpected = unorderable_list_difference(
|
||||
expected, actual)
|
||||
else:
|
||||
return self.assertSequenceEqual(expected, actual, msg=msg)
|
||||
|
||||
errors = []
|
||||
if missing:
|
||||
errors.append(
|
||||
'Expected, but missing:\n %s' % (safe_repr(missing), ),
|
||||
)
|
||||
if unexpected:
|
||||
errors.append(
|
||||
'Unexpected, but present:\n %s' % (safe_repr(unexpected), ),
|
||||
)
|
||||
if errors:
|
||||
standardMsg = '\n'.join(errors)
|
||||
self.fail(self._formatMessage(msg, standardMsg))
|
||||
@@ -0,0 +1,152 @@
|
||||
#
|
||||
# Module providing various facilities to other parts of the package
|
||||
#
|
||||
# billiard/util.py
|
||||
#
|
||||
# Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
|
||||
# Licensed to PSF under a Contributor Agreement.
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
|
||||
import errno
|
||||
import functools
|
||||
import atexit
|
||||
|
||||
from multiprocessing.util import ( # noqa
|
||||
_afterfork_registry,
|
||||
_afterfork_counter,
|
||||
_exit_function,
|
||||
_finalizer_registry,
|
||||
_finalizer_counter,
|
||||
Finalize,
|
||||
ForkAwareLocal,
|
||||
ForkAwareThreadLock,
|
||||
get_temp_dir,
|
||||
is_exiting,
|
||||
register_after_fork,
|
||||
_run_after_forkers,
|
||||
_run_finalizers,
|
||||
)
|
||||
|
||||
from .compat import get_errno
|
||||
|
||||
__all__ = [
|
||||
'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
|
||||
'log_to_stderr', 'get_temp_dir', 'register_after_fork',
|
||||
'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
|
||||
'SUBDEBUG', 'SUBWARNING',
|
||||
]
|
||||
|
||||
#
|
||||
# Logging
|
||||
#
|
||||
|
||||
NOTSET = 0
|
||||
SUBDEBUG = 5
|
||||
DEBUG = 10
|
||||
INFO = 20
|
||||
SUBWARNING = 25
|
||||
ERROR = 40
|
||||
|
||||
LOGGER_NAME = 'multiprocessing'
|
||||
DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
|
||||
|
||||
_logger = None
|
||||
_log_to_stderr = False
|
||||
|
||||
|
||||
def sub_debug(msg, *args, **kwargs):
|
||||
if _logger:
|
||||
_logger.log(SUBDEBUG, msg, *args, **kwargs)
|
||||
|
||||
|
||||
def debug(msg, *args, **kwargs):
|
||||
if _logger:
|
||||
_logger.log(DEBUG, msg, *args, **kwargs)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def info(msg, *args, **kwargs):
|
||||
if _logger:
|
||||
_logger.log(INFO, msg, *args, **kwargs)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def sub_warning(msg, *args, **kwargs):
|
||||
if _logger:
|
||||
_logger.log(SUBWARNING, msg, *args, **kwargs)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def error(msg, *args, **kwargs):
|
||||
if _logger:
|
||||
_logger.log(ERROR, msg, *args, **kwargs)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_logger():
|
||||
'''
|
||||
Returns logger used by multiprocessing
|
||||
'''
|
||||
global _logger
|
||||
import logging
|
||||
|
||||
logging._acquireLock()
|
||||
try:
|
||||
if not _logger:
|
||||
|
||||
_logger = logging.getLogger(LOGGER_NAME)
|
||||
_logger.propagate = 0
|
||||
logging.addLevelName(SUBDEBUG, 'SUBDEBUG')
|
||||
logging.addLevelName(SUBWARNING, 'SUBWARNING')
|
||||
|
||||
# XXX multiprocessing should cleanup before logging
|
||||
if hasattr(atexit, 'unregister'):
|
||||
atexit.unregister(_exit_function)
|
||||
atexit.register(_exit_function)
|
||||
else:
|
||||
atexit._exithandlers.remove((_exit_function, (), {}))
|
||||
atexit._exithandlers.append((_exit_function, (), {}))
|
||||
finally:
|
||||
logging._releaseLock()
|
||||
|
||||
return _logger
|
||||
|
||||
|
||||
def log_to_stderr(level=None):
|
||||
'''
|
||||
Turn on logging and add a handler which prints to stderr
|
||||
'''
|
||||
global _log_to_stderr
|
||||
import logging
|
||||
|
||||
logger = get_logger()
|
||||
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
if level:
|
||||
logger.setLevel(level)
|
||||
_log_to_stderr = True
|
||||
return _logger
|
||||
|
||||
|
||||
def _eintr_retry(func):
|
||||
'''
|
||||
Automatic retry after EINTR.
|
||||
'''
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapped(*args, **kwargs):
|
||||
while 1:
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except OSError as exc:
|
||||
if get_errno(exc) != errno.EINTR:
|
||||
raise
|
||||
return wrapped
|
||||
Reference in New Issue
Block a user