123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255 |
- #
- # Module which supports allocation of memory from an mmap
- #
- # multiprocessing/heap.py
- #
- # Copyright (c) 2006-2008, R Oudkerk
- # Licensed to PSF under a Contributor Agreement.
- #
- from __future__ import absolute_import
-
- import bisect
- import mmap
- import os
- import sys
- import threading
- import itertools
-
- from ._ext import _billiard, win32
- from .util import Finalize, info, get_temp_dir
- from .forking import assert_spawning
- from .reduction import ForkingPickler
-
- __all__ = ['BufferWrapper']
-
- try:
- maxsize = sys.maxsize
- except AttributeError:
- maxsize = sys.maxint
-
- #
- # Inheirtable class which wraps an mmap, and from which blocks can be allocated
- #
-
- if sys.platform == 'win32':
-
- class Arena(object):
-
- _counter = itertools.count()
-
- def __init__(self, size):
- self.size = size
- self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
- self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
- assert win32.GetLastError() == 0, 'tagname already in use'
- self._state = (self.size, self.name)
-
- def __getstate__(self):
- assert_spawning(self)
- return self._state
-
- def __setstate__(self, state):
- self.size, self.name = self._state = state
- self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
- assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
-
- else:
-
- class Arena(object):
-
- _counter = itertools.count()
-
- def __init__(self, size, fileno=-1):
- from .forking import _forking_is_enabled
- self.size = size
- self.fileno = fileno
- if fileno == -1 and not _forking_is_enabled:
- name = os.path.join(
- get_temp_dir(),
- 'pym-%d-%d' % (os.getpid(), next(self._counter)))
- self.fileno = os.open(
- name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600)
- os.unlink(name)
- os.ftruncate(self.fileno, size)
- self.buffer = mmap.mmap(self.fileno, self.size)
-
- def reduce_arena(a):
- if a.fileno == -1:
- raise ValueError('Arena is unpicklable because'
- 'forking was enabled when it was created')
- return Arena, (a.size, a.fileno)
-
- ForkingPickler.register(Arena, reduce_arena)
-
- #
- # Class allowing allocation of chunks of memory from arenas
- #
-
-
- class Heap(object):
-
- _alignment = 8
-
- def __init__(self, size=mmap.PAGESIZE):
- self._lastpid = os.getpid()
- self._lock = threading.Lock()
- self._size = size
- self._lengths = []
- self._len_to_seq = {}
- self._start_to_block = {}
- self._stop_to_block = {}
- self._allocated_blocks = set()
- self._arenas = []
- # list of pending blocks to free - see free() comment below
- self._pending_free_blocks = []
-
- @staticmethod
- def _roundup(n, alignment):
- # alignment must be a power of 2
- mask = alignment - 1
- return (n + mask) & ~mask
-
- def _malloc(self, size):
- # returns a large enough block -- it might be much larger
- i = bisect.bisect_left(self._lengths, size)
- if i == len(self._lengths):
- length = self._roundup(max(self._size, size), mmap.PAGESIZE)
- self._size *= 2
- info('allocating a new mmap of length %d', length)
- arena = Arena(length)
- self._arenas.append(arena)
- return (arena, 0, length)
- else:
- length = self._lengths[i]
- seq = self._len_to_seq[length]
- block = seq.pop()
- if not seq:
- del self._len_to_seq[length], self._lengths[i]
-
- (arena, start, stop) = block
- del self._start_to_block[(arena, start)]
- del self._stop_to_block[(arena, stop)]
- return block
-
- def _free(self, block):
- # free location and try to merge with neighbours
- (arena, start, stop) = block
-
- try:
- prev_block = self._stop_to_block[(arena, start)]
- except KeyError:
- pass
- else:
- start, _ = self._absorb(prev_block)
-
- try:
- next_block = self._start_to_block[(arena, stop)]
- except KeyError:
- pass
- else:
- _, stop = self._absorb(next_block)
-
- block = (arena, start, stop)
- length = stop - start
-
- try:
- self._len_to_seq[length].append(block)
- except KeyError:
- self._len_to_seq[length] = [block]
- bisect.insort(self._lengths, length)
-
- self._start_to_block[(arena, start)] = block
- self._stop_to_block[(arena, stop)] = block
-
- def _absorb(self, block):
- # deregister this block so it can be merged with a neighbour
- (arena, start, stop) = block
- del self._start_to_block[(arena, start)]
- del self._stop_to_block[(arena, stop)]
-
- length = stop - start
- seq = self._len_to_seq[length]
- seq.remove(block)
- if not seq:
- del self._len_to_seq[length]
- self._lengths.remove(length)
-
- return start, stop
-
- def _free_pending_blocks(self):
- # Free all the blocks in the pending list - called with the lock held
- while 1:
- try:
- block = self._pending_free_blocks.pop()
- except IndexError:
- break
- self._allocated_blocks.remove(block)
- self._free(block)
-
- def free(self, block):
- # free a block returned by malloc()
- # Since free() can be called asynchronously by the GC, it could happen
- # that it's called while self._lock is held: in that case,
- # self._lock.acquire() would deadlock (issue #12352). To avoid that, a
- # trylock is used instead, and if the lock can't be acquired
- # immediately, the block is added to a list of blocks to be freed
- # synchronously sometimes later from malloc() or free(), by calling
- # _free_pending_blocks() (appending and retrieving from a list is not
- # strictly thread-safe but under cPython it's atomic thanks
- # to the GIL).
- assert os.getpid() == self._lastpid
- if not self._lock.acquire(False):
- # can't aquire the lock right now, add the block to the list of
- # pending blocks to free
- self._pending_free_blocks.append(block)
- else:
- # we hold the lock
- try:
- self._free_pending_blocks()
- self._allocated_blocks.remove(block)
- self._free(block)
- finally:
- self._lock.release()
-
- def malloc(self, size):
- # return a block of right size (possibly rounded up)
- assert 0 <= size < maxsize
- if os.getpid() != self._lastpid:
- self.__init__() # reinitialize after fork
- self._lock.acquire()
- self._free_pending_blocks()
- try:
- size = self._roundup(max(size, 1), self._alignment)
- (arena, start, stop) = self._malloc(size)
- new_stop = start + size
- if new_stop < stop:
- self._free((arena, new_stop, stop))
- block = (arena, start, new_stop)
- self._allocated_blocks.add(block)
- return block
- finally:
- self._lock.release()
-
- #
- # Class representing a chunk of an mmap -- can be inherited
- #
-
-
- class BufferWrapper(object):
-
- _heap = Heap()
-
- def __init__(self, size):
- assert 0 <= size < maxsize
- block = BufferWrapper._heap.malloc(size)
- self._state = (block, size)
- Finalize(self, BufferWrapper._heap.free, args=(block,))
-
- def get_address(self):
- (arena, start, stop), size = self._state
- address, length = _billiard.address_of_buffer(arena.buffer)
- assert size <= length
- return address + start
-
- def get_size(self):
- return self._state[1]
|