# # Module implementing synchronization primitives # # multiprocessing/synchronize.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # from __future__ import absolute_import import itertools import os import signal import sys import threading from ._ext import _billiard, ensure_SemLock from .five import range, monotonic from .process import current_process from .util import Finalize, register_after_fork, debug from .forking import assert_spawning, Popen from .compat import bytes, closerange __all__ = [ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event', ] # Try to import the mp.synchronize module cleanly, if it fails # raise ImportError for platforms lacking a working sem_open implementation. # See issue 3770 ensure_SemLock() # # Constants # RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) SEM_VALUE_MAX = _billiard.SemLock.SEM_VALUE_MAX try: sem_unlink = _billiard.SemLock.sem_unlink except AttributeError: # pragma: no cover try: # Py3.4+ implements sem_unlink and the semaphore must be named from _multiprocessing import sem_unlink # noqa except ImportError: sem_unlink = None # noqa # # Base class for semaphores and mutexes; wraps `_billiard.SemLock` # def _semname(sl): try: return sl.name except AttributeError: pass class SemLock(object): _counter = itertools.count() def __init__(self, kind, value, maxvalue): from .forking import _forking_is_enabled unlink_immediately = _forking_is_enabled or sys.platform == 'win32' if sem_unlink: sl = self._semlock = _billiard.SemLock( kind, value, maxvalue, self._make_name(), unlink_immediately) else: sl = self._semlock = _billiard.SemLock(kind, value, maxvalue) debug('created semlock with handle %s', sl.handle) self._make_methods() if sem_unlink: if sys.platform != 'win32': def _after_fork(obj): obj._semlock._after_fork() register_after_fork(self, _after_fork) if _semname(self._semlock) is not None: # We only get here if we are on Unix with forking # disabled. When the object is garbage collected or the # process shuts down we unlink the semaphore name Finalize(self, sem_unlink, (self._semlock.name,), exitpriority=0) # In case of abnormal termination unlink semaphore name _cleanup_semaphore_if_leaked(self._semlock.name) def _make_methods(self): self.acquire = self._semlock.acquire self.release = self._semlock.release def __enter__(self): return self._semlock.__enter__() def __exit__(self, *args): return self._semlock.__exit__(*args) def __getstate__(self): assert_spawning(self) sl = self._semlock state = (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue) try: state += (sl.name, ) except AttributeError: pass return state def __setstate__(self, state): self._semlock = _billiard.SemLock._rebuild(*state) debug('recreated blocker with handle %r', state[0]) self._make_methods() @staticmethod def _make_name(): return '/%s-%s-%s' % (current_process()._semprefix, os.getpid(), next(SemLock._counter)) class Semaphore(SemLock): def __init__(self, value=1): SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX) def get_value(self): return self._semlock._get_value() def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '' % value class BoundedSemaphore(Semaphore): def __init__(self, value=1): SemLock.__init__(self, SEMAPHORE, value, value) def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '' % \ (value, self._semlock.maxvalue) class Lock(SemLock): ''' Non-recursive lock. ''' def __init__(self): SemLock.__init__(self, SEMAPHORE, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): name = current_process().name if threading.currentThread().name != 'MainThread': name += '|' + threading.currentThread().name elif self._semlock._get_value() == 1: name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' else: name = 'SomeOtherProcess' except Exception: name = 'unknown' return '' % name class RLock(SemLock): ''' Recursive lock ''' def __init__(self): SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): name = current_process().name if threading.currentThread().name != 'MainThread': name += '|' + threading.currentThread().name count = self._semlock._count() elif self._semlock._get_value() == 1: name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' else: name, count = 'SomeOtherProcess', 'nonzero' except Exception: name, count = 'unknown', 'unknown' return '' % (name, count) class Condition(object): ''' Condition variable ''' def __init__(self, lock=None): self._lock = lock or RLock() self._sleeping_count = Semaphore(0) self._woken_count = Semaphore(0) self._wait_semaphore = Semaphore(0) self._make_methods() def __getstate__(self): assert_spawning(self) return (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) def __setstate__(self, state): (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) = state self._make_methods() def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def _make_methods(self): self.acquire = self._lock.acquire self.release = self._lock.release def __repr__(self): try: num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: num_waiters = 'unkown' return '' % (self._lock, num_waiters) def wait(self, timeout=None): assert self._lock._semlock._is_mine(), \ 'must acquire() condition before using wait()' # indicate that this thread is going to sleep self._sleeping_count.release() # release lock count = self._lock._semlock._count() for i in range(count): self._lock.release() try: # wait for notification or timeout ret = self._wait_semaphore.acquire(True, timeout) finally: # indicate that this thread has woken self._woken_count.release() # reacquire lock for i in range(count): self._lock.acquire() return ret def notify(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res if self._sleeping_count.acquire(False): # try grabbing a sleeper self._wait_semaphore.release() # wake up one sleeper self._woken_count.acquire() # wait for sleeper to wake # rezero _wait_semaphore in case a timeout just happened self._wait_semaphore.acquire(False) def notify_all(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify*() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res sleepers = 0 while self._sleeping_count.acquire(False): self._wait_semaphore.release() # wake up one sleeper sleepers += 1 if sleepers: for i in range(sleepers): self._woken_count.acquire() # wait for a sleeper to wake # rezero wait_semaphore in case some timeouts just happened while self._wait_semaphore.acquire(False): pass def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = monotonic() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - monotonic() if waittime <= 0: break self.wait(waittime) result = predicate() return result class Event(object): def __init__(self): self._cond = Condition(Lock()) self._flag = Semaphore(0) def is_set(self): self._cond.acquire() try: if self._flag.acquire(False): self._flag.release() return True return False finally: self._cond.release() def set(self): self._cond.acquire() try: self._flag.acquire(False) self._flag.release() self._cond.notify_all() finally: self._cond.release() def clear(self): self._cond.acquire() try: self._flag.acquire(False) finally: self._cond.release() def wait(self, timeout=None): self._cond.acquire() try: if self._flag.acquire(False): self._flag.release() else: self._cond.wait(timeout) if self._flag.acquire(False): self._flag.release() return True return False finally: self._cond.release() if sys.platform != 'win32': # # Protection against unlinked semaphores if the program ends abnormally # and forking has been disabled. # def _cleanup_semaphore_if_leaked(name): name = name.encode('ascii') + bytes('\0', 'ascii') if len(name) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError('name too long') fd = _get_unlinkfd() bits = os.write(fd, name) assert bits == len(name) def _get_unlinkfd(): cp = current_process() if cp._unlinkfd is None: r, w = os.pipe() pid = os.fork() if pid == 0: try: from setproctitle import setproctitle setproctitle("[sem_cleanup for %r]" % cp.pid) except: pass # Fork a process which will survive until all other processes # which have a copy of the write end of the pipe have exited. # The forked process just collects names of semaphores until # EOF is indicated. Then it tries unlinking all the names it # has collected. _collect_names_then_unlink(r) os._exit(0) os.close(r) cp._unlinkfd = w return cp._unlinkfd def _collect_names_then_unlink(r): # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) # close all fds except r try: MAXFD = os.sysconf("SC_OPEN_MAX") except: MAXFD = 256 closerange(0, r) closerange(r + 1, MAXFD) # collect data written to pipe data = [] while 1: try: s = os.read(r, 512) except: # XXX IO lock might be held at fork, so don't try # printing unexpected exception - see issue 6721 pass else: if not s: break data.append(s) # attempt to unlink each collected name for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')): try: sem_unlink(name.decode('ascii')) except: # XXX IO lock might be held at fork, so don't try # printing unexpected exception - see issue 6721 pass