123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449 |
- #
- # Module implementing synchronization primitives
- #
- # multiprocessing/synchronize.py
- #
- # Copyright (c) 2006-2008, R Oudkerk
- # Licensed to PSF under a Contributor Agreement.
- #
- from __future__ import absolute_import
-
- import itertools
- import os
- import signal
- import sys
- import threading
-
-
- from ._ext import _billiard, ensure_SemLock
- from .five import range, monotonic
- from .process import current_process
- from .util import Finalize, register_after_fork, debug
- from .forking import assert_spawning, Popen
- from .compat import bytes, closerange
-
- __all__ = [
- 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event',
- ]
-
- # Try to import the mp.synchronize module cleanly, if it fails
- # raise ImportError for platforms lacking a working sem_open implementation.
- # See issue 3770
- ensure_SemLock()
-
- #
- # Constants
- #
-
- RECURSIVE_MUTEX, SEMAPHORE = list(range(2))
- SEM_VALUE_MAX = _billiard.SemLock.SEM_VALUE_MAX
-
- try:
- sem_unlink = _billiard.SemLock.sem_unlink
- except AttributeError: # pragma: no cover
- try:
- # Py3.4+ implements sem_unlink and the semaphore must be named
- from _multiprocessing import sem_unlink # noqa
- except ImportError:
- sem_unlink = None # noqa
-
- #
- # Base class for semaphores and mutexes; wraps `_billiard.SemLock`
- #
-
-
- def _semname(sl):
- try:
- return sl.name
- except AttributeError:
- pass
-
-
- class SemLock(object):
- _counter = itertools.count()
-
- def __init__(self, kind, value, maxvalue):
- from .forking import _forking_is_enabled
- unlink_immediately = _forking_is_enabled or sys.platform == 'win32'
- if sem_unlink:
- sl = self._semlock = _billiard.SemLock(
- kind, value, maxvalue, self._make_name(), unlink_immediately)
- else:
- sl = self._semlock = _billiard.SemLock(kind, value, maxvalue)
-
- debug('created semlock with handle %s', sl.handle)
- self._make_methods()
-
- if sem_unlink:
-
- if sys.platform != 'win32':
- def _after_fork(obj):
- obj._semlock._after_fork()
- register_after_fork(self, _after_fork)
-
- if _semname(self._semlock) is not None:
- # We only get here if we are on Unix with forking
- # disabled. When the object is garbage collected or the
- # process shuts down we unlink the semaphore name
- Finalize(self, sem_unlink, (self._semlock.name,),
- exitpriority=0)
- # In case of abnormal termination unlink semaphore name
- _cleanup_semaphore_if_leaked(self._semlock.name)
-
- def _make_methods(self):
- self.acquire = self._semlock.acquire
- self.release = self._semlock.release
-
- def __enter__(self):
- return self._semlock.__enter__()
-
- def __exit__(self, *args):
- return self._semlock.__exit__(*args)
-
- def __getstate__(self):
- assert_spawning(self)
- sl = self._semlock
- state = (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)
- try:
- state += (sl.name, )
- except AttributeError:
- pass
- return state
-
- def __setstate__(self, state):
- self._semlock = _billiard.SemLock._rebuild(*state)
- debug('recreated blocker with handle %r', state[0])
- self._make_methods()
-
- @staticmethod
- def _make_name():
- return '/%s-%s-%s' % (current_process()._semprefix,
- os.getpid(), next(SemLock._counter))
-
-
- class Semaphore(SemLock):
-
- def __init__(self, value=1):
- SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
-
- def get_value(self):
- return self._semlock._get_value()
-
- def __repr__(self):
- try:
- value = self._semlock._get_value()
- except Exception:
- value = 'unknown'
- return '<Semaphore(value=%s)>' % value
-
-
- class BoundedSemaphore(Semaphore):
-
- def __init__(self, value=1):
- SemLock.__init__(self, SEMAPHORE, value, value)
-
- def __repr__(self):
- try:
- value = self._semlock._get_value()
- except Exception:
- value = 'unknown'
- return '<BoundedSemaphore(value=%s, maxvalue=%s)>' % \
- (value, self._semlock.maxvalue)
-
-
- class Lock(SemLock):
- '''
- Non-recursive lock.
- '''
-
- def __init__(self):
- SemLock.__init__(self, SEMAPHORE, 1, 1)
-
- def __repr__(self):
- try:
- if self._semlock._is_mine():
- name = current_process().name
- if threading.currentThread().name != 'MainThread':
- name += '|' + threading.currentThread().name
- elif self._semlock._get_value() == 1:
- name = 'None'
- elif self._semlock._count() > 0:
- name = 'SomeOtherThread'
- else:
- name = 'SomeOtherProcess'
- except Exception:
- name = 'unknown'
- return '<Lock(owner=%s)>' % name
-
-
- class RLock(SemLock):
- '''
- Recursive lock
- '''
-
- def __init__(self):
- SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
-
- def __repr__(self):
- try:
- if self._semlock._is_mine():
- name = current_process().name
- if threading.currentThread().name != 'MainThread':
- name += '|' + threading.currentThread().name
- count = self._semlock._count()
- elif self._semlock._get_value() == 1:
- name, count = 'None', 0
- elif self._semlock._count() > 0:
- name, count = 'SomeOtherThread', 'nonzero'
- else:
- name, count = 'SomeOtherProcess', 'nonzero'
- except Exception:
- name, count = 'unknown', 'unknown'
- return '<RLock(%s, %s)>' % (name, count)
-
-
- class Condition(object):
- '''
- Condition variable
- '''
-
- def __init__(self, lock=None):
- self._lock = lock or RLock()
- self._sleeping_count = Semaphore(0)
- self._woken_count = Semaphore(0)
- self._wait_semaphore = Semaphore(0)
- self._make_methods()
-
- def __getstate__(self):
- assert_spawning(self)
- return (self._lock, self._sleeping_count,
- self._woken_count, self._wait_semaphore)
-
- def __setstate__(self, state):
- (self._lock, self._sleeping_count,
- self._woken_count, self._wait_semaphore) = state
- self._make_methods()
-
- def __enter__(self):
- return self._lock.__enter__()
-
- def __exit__(self, *args):
- return self._lock.__exit__(*args)
-
- def _make_methods(self):
- self.acquire = self._lock.acquire
- self.release = self._lock.release
-
- def __repr__(self):
- try:
- num_waiters = (self._sleeping_count._semlock._get_value() -
- self._woken_count._semlock._get_value())
- except Exception:
- num_waiters = 'unkown'
- return '<Condition(%s, %s)>' % (self._lock, num_waiters)
-
- def wait(self, timeout=None):
- assert self._lock._semlock._is_mine(), \
- 'must acquire() condition before using wait()'
-
- # indicate that this thread is going to sleep
- self._sleeping_count.release()
-
- # release lock
- count = self._lock._semlock._count()
- for i in range(count):
- self._lock.release()
-
- try:
- # wait for notification or timeout
- ret = self._wait_semaphore.acquire(True, timeout)
- finally:
- # indicate that this thread has woken
- self._woken_count.release()
-
- # reacquire lock
- for i in range(count):
- self._lock.acquire()
- return ret
-
- def notify(self):
- assert self._lock._semlock._is_mine(), 'lock is not owned'
- assert not self._wait_semaphore.acquire(False)
-
- # to take account of timeouts since last notify() we subtract
- # woken_count from sleeping_count and rezero woken_count
- while self._woken_count.acquire(False):
- res = self._sleeping_count.acquire(False)
- assert res
-
- if self._sleeping_count.acquire(False): # try grabbing a sleeper
- self._wait_semaphore.release() # wake up one sleeper
- self._woken_count.acquire() # wait for sleeper to wake
-
- # rezero _wait_semaphore in case a timeout just happened
- self._wait_semaphore.acquire(False)
-
- def notify_all(self):
- assert self._lock._semlock._is_mine(), 'lock is not owned'
- assert not self._wait_semaphore.acquire(False)
-
- # to take account of timeouts since last notify*() we subtract
- # woken_count from sleeping_count and rezero woken_count
- while self._woken_count.acquire(False):
- res = self._sleeping_count.acquire(False)
- assert res
-
- sleepers = 0
- while self._sleeping_count.acquire(False):
- self._wait_semaphore.release() # wake up one sleeper
- sleepers += 1
-
- if sleepers:
- for i in range(sleepers):
- self._woken_count.acquire() # wait for a sleeper to wake
-
- # rezero wait_semaphore in case some timeouts just happened
- while self._wait_semaphore.acquire(False):
- pass
-
- def wait_for(self, predicate, timeout=None):
- result = predicate()
- if result:
- return result
- if timeout is not None:
- endtime = monotonic() + timeout
- else:
- endtime = None
- waittime = None
- while not result:
- if endtime is not None:
- waittime = endtime - monotonic()
- if waittime <= 0:
- break
- self.wait(waittime)
- result = predicate()
- return result
-
-
- class Event(object):
-
- def __init__(self):
- self._cond = Condition(Lock())
- self._flag = Semaphore(0)
-
- def is_set(self):
- self._cond.acquire()
- try:
- if self._flag.acquire(False):
- self._flag.release()
- return True
- return False
- finally:
- self._cond.release()
-
- def set(self):
- self._cond.acquire()
- try:
- self._flag.acquire(False)
- self._flag.release()
- self._cond.notify_all()
- finally:
- self._cond.release()
-
- def clear(self):
- self._cond.acquire()
- try:
- self._flag.acquire(False)
- finally:
- self._cond.release()
-
- def wait(self, timeout=None):
- self._cond.acquire()
- try:
- if self._flag.acquire(False):
- self._flag.release()
- else:
- self._cond.wait(timeout)
-
- if self._flag.acquire(False):
- self._flag.release()
- return True
- return False
- finally:
- self._cond.release()
-
-
- if sys.platform != 'win32':
- #
- # Protection against unlinked semaphores if the program ends abnormally
- # and forking has been disabled.
- #
-
- def _cleanup_semaphore_if_leaked(name):
- name = name.encode('ascii') + bytes('\0', 'ascii')
- if len(name) > 512:
- # posix guarantees that writes to a pipe of less than PIPE_BUF
- # bytes are atomic, and that PIPE_BUF >= 512
- raise ValueError('name too long')
- fd = _get_unlinkfd()
- bits = os.write(fd, name)
- assert bits == len(name)
-
- def _get_unlinkfd():
- cp = current_process()
- if cp._unlinkfd is None:
- r, w = os.pipe()
- pid = os.fork()
- if pid == 0:
- try:
- from setproctitle import setproctitle
- setproctitle("[sem_cleanup for %r]" % cp.pid)
- except:
- pass
-
- # Fork a process which will survive until all other processes
- # which have a copy of the write end of the pipe have exited.
- # The forked process just collects names of semaphores until
- # EOF is indicated. Then it tries unlinking all the names it
- # has collected.
- _collect_names_then_unlink(r)
- os._exit(0)
- os.close(r)
- cp._unlinkfd = w
- return cp._unlinkfd
-
- def _collect_names_then_unlink(r):
- # protect the process from ^C and "killall python" etc
- signal.signal(signal.SIGINT, signal.SIG_IGN)
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
-
- # close all fds except r
- try:
- MAXFD = os.sysconf("SC_OPEN_MAX")
- except:
- MAXFD = 256
- closerange(0, r)
- closerange(r + 1, MAXFD)
-
- # collect data written to pipe
- data = []
- while 1:
- try:
- s = os.read(r, 512)
- except:
- # XXX IO lock might be held at fork, so don't try
- # printing unexpected exception - see issue 6721
- pass
- else:
- if not s:
- break
- data.append(s)
-
- # attempt to unlink each collected name
- for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')):
- try:
- sem_unlink(name.decode('ascii'))
- except:
- # XXX IO lock might be held at fork, so don't try
- # printing unexpected exception - see issue 6721
- pass
|