12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595 |
- ##############################################################################
- #
- # Copyright 2011 Zope Foundation and Contributors.
- # All Rights Reserved.
- #
- # This software is subject to the provisions of the Zope Public License,
- # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
- # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
- # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
- # FOR A PARTICULAR PURPOSE.
- #
- ##############################################################################
- """Python BTree implementation
- """
-
- from struct import Struct
- from struct import error as struct_error
- from operator import index
-
- from persistent import Persistent
-
- from .Interfaces import BTreesConflictError
- from ._compat import PY3
- from ._compat import compare
- from ._compat import int_types
- from ._compat import xrange
-
-
- _marker = object()
-
-
- class _Base(Persistent):
-
- __slots__ = ()
- _key_type = list
-
- def __init__(self, items=None):
- self.clear()
- if items:
- self.update(items)
-
- try:
- # Detect the presence of the C extensions.
- # If they're NOT around, we don't need to do any of the
- # special pickle support to make Python versions look like
- # C---we just rename the classes. By not defining these methods,
- # we can (theoretically) avoid a bit of a slowdown.
- # If the C extensions are around, we do need these methods, but
- # these classes are unlikely to be used in production anyway.
- __import__('BTrees._OOBTree')
- except ImportError: # pragma: no cover
- pass
- else:
- def __reduce__(self):
- # Swap out the type constructor for the C version, if present.
- func, typ_gna, state = Persistent.__reduce__(self)
- # We ignore the returned type altogether in favor of
- # our calculated class (which allows subclasses but replaces our exact
- # type with the C equivalent)
- typ = self.__class__
- gna = typ_gna[1:]
- return (func, (typ,) + gna, state)
-
- @property
- def __class__(self):
- type_self = type(self)
- return type_self._BTree_reduce_as if type_self._BTree_reduce_up_bound is type_self else type_self
-
- @property
- def _BTree_reduce_as(self):
- # Return the pickle replacement class for this object.
- # If the C extensions are available, this will be the
- # C type (setup by _fix_pickle), otherwise it will be the real
- # type of this object.
- # This implementation is replaced by _fix_pickle and exists for
- # testing purposes.
- return type(self) # pragma: no cover
-
- _BTree_reduce_up_bound = _BTree_reduce_as
-
- class _BucketBase(_Base):
-
- __slots__ = ('_keys',
- '_next',
- '_to_key',
- )
-
- def clear(self):
- self._keys = self._key_type()
- self._next = None
-
- def __len__(self):
- return len(self._keys)
-
- @property
- def size(self):
- return len(self._keys)
-
- def _deleteNextBucket(self):
- next = self._next
- if next is not None:
- self._next = next._next
-
- def _search(self, key):
- # Return non-negative index on success
- # return -(insertion_index + 1) on fail
- low = 0
- keys = self._keys
- high = len(keys)
- while low < high:
- i = (low + high) // 2
- k = keys[i]
- if k is key or k == key:
- return i
-
- if compare(k, key) < 0:
- low = i + 1
- else:
- high = i
- return -1 - low
-
- def minKey(self, key=_marker):
- if key is _marker or key is None:
- return self._keys[0]
- key = self._to_key(key)
- index = self._search(key)
- if index >= 0:
- return key
- index = -index - 1
- if index < len(self._keys):
- return self._keys[index]
- else:
- raise ValueError("no key satisfies the conditions")
-
- def maxKey(self, key=_marker):
- if key is _marker or key is None:
- return self._keys[-1]
- key = self._to_key(key)
- index = self._search(key)
- if index >= 0:
- return key
- else:
- index = -index-1
- if index:
- return self._keys[index-1]
- else:
- raise ValueError("no key satisfies the conditions")
-
- def _range(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- if min is _marker or min is None:
- start = 0
- if excludemin:
- start = 1
- else:
- min = self._to_key(min)
- start = self._search(min)
- if start >= 0:
- if excludemin:
- start += 1
- else:
- start = -start - 1
- if max is _marker or max is None:
- end = len(self._keys)
- if excludemax:
- end -= 1
- else:
- max = self._to_key(max)
- end = self._search(max)
- if end >= 0:
- if not excludemax:
- end += 1
- else:
- end = -end - 1
-
- return start, end
-
- def keys(self, *args, **kw):
- start, end = self._range(*args, **kw)
- return self._keys[start:end]
-
- def iterkeys(self, *args, **kw):
- if not (args or kw):
- return iter(self._keys)
- keys = self._keys
- return (keys[i] for i in xrange(*self._range(*args, **kw)))
-
- def __iter__(self):
- return iter(self._keys)
-
- def __contains__(self, key):
- return (self._search(self._to_key(key)) >= 0)
-
- has_key = __contains__
-
- def _repr_helper(self, items):
- type_self = type(self)
- mod = type_self.__module__
- name = type_self.__name__
- name = name[:-2] if name.endswith("Py") else name
- return "%s.%s(%r)" % (mod, name, items)
-
- class _SetIteration(object):
-
- __slots__ = ('to_iterate',
- 'useValues',
- '_iter',
- 'active',
- 'position',
- 'key',
- 'value',
- )
-
-
- def __init__(self, to_iterate, useValues=False, default=None):
- if to_iterate is None:
- to_iterate = ()
- self.to_iterate = to_iterate
- if useValues:
- try:
- itmeth = to_iterate.iteritems
- except AttributeError:
- if PY3 and isinstance(to_iterate, dict): #pragma no cover Py3k
- itmeth = to_iterate.items().__iter__
- else:
- itmeth = to_iterate.__iter__
- useValues = False
- else:
- self.value = None
- else:
- itmeth = to_iterate.__iter__
-
- self.useValues = useValues
- self._iter = itmeth()
- self.active = True
- self.position = 0
- self.key = _marker
- self.value = default
- self.advance()
-
- def advance(self):
- try:
- if self.useValues:
- self.key, self.value = next(self._iter)
- else:
- self.key = next(self._iter)
- self.position += 1
- except StopIteration:
- self.active = False
- self.position = -1
-
- return self
-
- _object_lt = getattr(object, '__lt__', _marker)
-
- def _no_default_comparison(key):
- # Enforce test that key has non-default comparison.
- if key is None:
- return
- if type(key) is object:
- raise TypeError("Can't use object() as keys")
- lt = getattr(key, '__lt__', None)
- if lt is not None:
- # CPython 3.x follows PEP 252, defining '__objclass__'
- if getattr(lt, '__objclass__', None) is object:
- lt = None # pragma: no cover Py3k
- # PyPy3 doesn't follow PEP 252, but defines '__func__'
- elif getattr(lt, '__func__', None) is _object_lt:
- lt = None # pragma: no cover PyPy3
- if (lt is None and
- getattr(key, '__cmp__', None) is None):
- raise TypeError("Object has default comparison")
-
- class Bucket(_BucketBase):
-
- __slots__ = ()
- _value_type = list
- _to_value = lambda self, x: x
- VALUE_SAME_CHECK = False
-
- def setdefault(self, key, value):
- key, value = self._to_key(key), self._to_value(value)
- status, value = self._set(key, value, True)
- return value
-
- def pop(self, key, default=_marker):
- try:
- status, value = self._del(self._to_key(key))
- except KeyError:
- if default is _marker:
- raise
- return default
- else:
- return value
-
- def update(self, items):
- if hasattr(items, 'iteritems'):
- items = items.iteritems()
- elif hasattr(items, 'items'):
- items = items.items()
-
- _si = self.__setitem__
- try:
- for key, value in items:
- _si(key, value)
- except ValueError:
- raise TypeError('items must be a sequence of 2-tuples')
-
- def __setitem__(self, key, value):
- _no_default_comparison(key)
- self._set(self._to_key(key), self._to_value(value))
-
- def __delitem__(self, key):
- self._del(self._to_key(key))
-
- def clear(self):
- _BucketBase.clear(self)
- self._values = self._value_type()
-
- def get(self, key, default=None):
- index = self._search(self._to_key(key))
- if index < 0:
- return default
- return self._values[index]
-
- def __getitem__(self, key):
- index = self._search(self._to_key(key))
- if index < 0:
- raise KeyError(key)
- return self._values[index]
-
- def _set(self, key, value, ifunset=False):
- """Set a value
-
- Return: status, value
-
- Status is:
- None if no change
- 0 if change, but not size change
- 1 if change and size change
- """
- index = self._search(key)
- if index >= 0:
- if (ifunset or
- self.VALUE_SAME_CHECK and value == self._values[index]
- ):
- return None, self._values[index]
- self._p_changed = True
- self._values[index] = value
- return 0, value
- else:
- self._p_changed = True
- index = -index - 1
- self._keys.insert(index, key)
- self._values.insert(index, value)
- return 1, value
-
- def _del(self, key):
- index = self._search(key)
- if index >= 0:
- self._p_changed = True
- del self._keys[index]
- return 0, self._values.pop(index)
- raise KeyError(key)
-
- def _split(self, index=-1):
- if index < 0 or index >= len(self._keys):
- index = len(self._keys) // 2
- new_instance = type(self)()
- new_instance._keys = self._keys[index:]
- new_instance._values = self._values[index:]
- del self._keys[index:]
- del self._values[index:]
- new_instance._next = self._next
- self._next = new_instance
- return new_instance
-
- def values(self, *args, **kw):
- start, end = self._range(*args, **kw)
- return self._values[start:end]
-
- def itervalues(self, *args, **kw):
- values = self._values
- return (values[i] for i in xrange(*self._range(*args, **kw)))
-
- def items(self, *args, **kw):
- keys = self._keys
- values = self._values
- return [(keys[i], values[i])
- for i in xrange(*self._range(*args, **kw))]
-
- def iteritems(self, *args, **kw):
- keys = self._keys
- values = self._values
- return ((keys[i], values[i])
- for i in xrange(*self._range(*args, **kw)))
-
- def __getstate__(self):
- keys = self._keys
- values = self._values
- data = []
- for i in range(len(keys)):
- data.append(keys[i])
- data.append(values[i])
- data = tuple(data)
-
- if self._next is not None:
- return data, self._next
- return (data, )
-
- def __setstate__(self, state):
- if not isinstance(state[0], tuple):
- raise TypeError("tuple required for first state element")
-
- self.clear()
- if len(state) == 2:
- state, self._next = state
- else:
- self._next = None
- state = state[0]
-
- keys = self._keys
- values = self._values
- for i in range(0, len(state), 2):
- keys.append(state[i])
- values.append(state[i+1])
-
- def _p_resolveConflict(self, s_old, s_com, s_new):
- b_old = type(self)()
- if s_old is not None:
- b_old.__setstate__(s_old)
- b_com = type(self)()
- if s_com is not None:
- b_com.__setstate__(s_com)
- b_new = type(self)()
- if s_new is not None:
- b_new.__setstate__(s_new)
- if (b_com._next != b_old._next or
- b_new._next != b_old._next):
- raise BTreesConflictError(-1, -1, -1, 0)
-
- if not b_com or not b_new:
- raise BTreesConflictError(-1, -1, -1, 12)
-
- i_old = _SetIteration(b_old, True)
- i_com = _SetIteration(b_com, True)
- i_new = _SetIteration(b_new, True)
-
- def merge_error(reason):
- return BTreesConflictError(
- i_old.position, i_com.position, i_new.position, reason)
-
- result = type(self)()
-
- def merge_output(it):
- result._keys.append(it.key)
- result._values.append(it.value)
- it.advance()
-
- while i_old.active and i_com.active and i_new.active:
- cmpOC = compare(i_old.key, i_com.key)
- cmpON = compare(i_old.key, i_new.key)
- if cmpOC == 0:
- if cmpON == 0:
- if i_com.value == i_old.value:
- result[i_old.key] = i_new.value
- elif i_new.value == i_old.value:
- result[i_old.key] = i_com.value
- else:
- raise merge_error(1)
- i_old.advance()
- i_com.advance()
- i_new.advance()
- elif (cmpON > 0): # insert in new
- merge_output(i_new)
- elif i_old.value == i_com.value: # deleted new
- if i_new.position == 1:
- # Deleted the first item. This will modify the
- # parent node, so we don't know if merging will be
- # safe
- raise merge_error(13)
- i_old.advance()
- i_com.advance()
- else:
- raise merge_error(2)
- elif cmpON == 0:
- if cmpOC > 0: # insert committed
- merge_output(i_com)
- elif i_old.value == i_new.value: # delete committed
- if i_com.position == 1:
- # Deleted the first item. This will modify the
- # parent node, so we don't know if merging will be
- # safe
- raise merge_error(13)
- i_old.advance()
- i_new.advance()
- else:
- raise merge_error(3)
- else: # both keys changed
- cmpCN = compare(i_com.key, i_new.key)
- if cmpCN == 0: # dueling insert
- raise merge_error(4)
- if cmpOC > 0: # insert committed
- if cmpCN > 0: # insert i_new first
- merge_output(i_new)
- else:
- merge_output(i_com)
- elif cmpON > 0: # insert i_new
- merge_output(i_new)
- else:
- raise merge_error(5) # both deleted same key
-
- while i_com.active and i_new.active: # new inserts
- cmpCN = compare(i_com.key, i_new.key)
- if cmpCN == 0:
- raise merge_error(6) # dueling insert
- if cmpCN > 0: # insert new
- merge_output(i_new)
- else: # insert committed
- merge_output(i_com)
-
- while i_old.active and i_com.active: # new deletes rest of original
- cmpOC = compare(i_old.key, i_com.key)
- if cmpOC > 0: # insert committed
- merge_output(i_com)
- elif cmpOC == 0 and (i_old.value == i_com.value): # del in new
- i_old.advance()
- i_com.advance()
- else: # dueling deletes or delete and change
- raise merge_error(7)
-
- while i_old.active and i_new.active:
- # committed deletes rest of original
- cmpON = compare(i_old.key, i_new.key)
- if cmpON > 0: # insert new
- merge_output(i_new)
- elif cmpON == 0 and (i_old.value == i_new.value):
- # deleted in committed
- i_old.advance()
- i_new.advance()
- else: # dueling deletes or delete and change
- raise merge_error(8)
-
- if i_old.active: # dueling deletes
- raise merge_error(9)
-
- while i_com.active:
- merge_output(i_com)
-
- while i_new.active:
- merge_output(i_new)
-
- if len(result._keys) == 0: #pragma: no cover
- # If the output bucket is empty, conflict resolution doesn't have
- # enough info to unlink it from its containing BTree correctly.
- #
- # XXX TS, 2012-11-16: I don't think this is possible
- #
- raise merge_error(10)
-
- result._next = b_old._next
- return result.__getstate__()
-
- def __repr__(self):
- return self._repr_helper(self.items())
-
- class Set(_BucketBase):
-
- __slots__ = ()
-
- def add(self, key):
- return self._set(self._to_key(key))[0]
-
- insert = add
-
- def remove(self, key):
- self._del(self._to_key(key))
-
- def update(self, items):
- add = self.add
- for i in items:
- add(i)
-
- def __getstate__(self):
- data = tuple(self._keys)
- if self._next is not None:
- return data, self._next
- return (data, )
-
- def __setstate__(self, state):
- if not isinstance(state[0], tuple):
- raise TypeError('tuple required for first state element')
-
- self.clear()
- if len(state) == 2:
- state, self._next = state
- else:
- self._next = None
- state = state[0]
-
- self._keys.extend(state)
-
-
- def _set(self, key, value=None, ifunset=False):
- index = self._search(key)
- if index < 0:
- index = -index - 1
- self._p_changed = True
- self._keys.insert(index, key)
- return True, None
- return False, None
-
- def _del(self, key):
- index = self._search(key)
- if index >= 0:
- self._p_changed = True
- del self._keys[index]
- return 0, 0
- raise KeyError(key)
-
- def __getitem__(self, i):
- return self._keys[i]
-
- def _split(self, index=-1):
- if index < 0 or index >= len(self._keys):
- index = len(self._keys) // 2
- new_instance = type(self)()
- new_instance._keys = self._keys[index:]
- del self._keys[index:]
- new_instance._next = self._next
- self._next = new_instance
- return new_instance
-
- def _p_resolveConflict(self, s_old, s_com, s_new):
-
- b_old = type(self)()
- if s_old is not None:
- b_old.__setstate__(s_old)
- b_com = type(self)()
- if s_com is not None:
- b_com.__setstate__(s_com)
- b_new = type(self)()
- if s_new is not None:
- b_new.__setstate__(s_new)
-
- if (b_com._next != b_old._next or
- b_new._next != b_old._next): # conflict: com or new changed _next
- raise BTreesConflictError(-1, -1, -1, 0)
-
- if not b_com or not b_new: # conflict: com or new empty
- raise BTreesConflictError(-1, -1, -1, 12)
-
- i_old = _SetIteration(b_old, True)
- i_com = _SetIteration(b_com, True)
- i_new = _SetIteration(b_new, True)
-
- def merge_error(reason):
- return BTreesConflictError(
- i_old.position, i_com.position, i_new.position, reason)
-
- result = type(self)()
-
- def merge_output(it):
- result._keys.append(it.key)
- it.advance()
-
- while i_old.active and i_com.active and i_new.active:
- cmpOC = compare(i_old.key, i_com.key)
- cmpON = compare(i_old.key, i_new.key)
- if cmpOC == 0:
- if cmpON == 0: # all match
- merge_output(i_old)
- i_com.advance()
- i_new.advance()
- elif cmpON > 0: # insert in new
- merge_output(i_new)
- else: # deleted new
- if i_new.position == 1:
- # Deleted the first item. This will modify the
- # parent node, so we don't know if merging will be
- # safe
- raise merge_error(13)
- i_old.advance()
- i_com.advance()
- elif cmpON == 0:
- if cmpOC > 0: # insert committed
- merge_output(i_com)
- else: # delete committed
- if i_com.position == 1:
- # Deleted the first item. This will modify the
- # parent node, so we don't know if merging will be
- # safe
- raise merge_error(13)
- i_old.advance()
- i_new.advance()
- else: # both com and new keys changed
- cmpCN = compare(i_com.key, i_new.key)
- if cmpCN == 0: # both inserted same key
- raise merge_error(4)
- if cmpOC > 0: # insert committed
- if cmpCN > 0: # insert i_new first
- merge_output(i_new)
- else:
- merge_output(i_com)
- elif cmpON > 0: # insert i_new
- merge_output(i_new)
- else: # both com and new deleted same key
- raise merge_error(5)
-
- while i_com.active and i_new.active: # new inserts
- cmpCN = compare(i_com.key, i_new.key)
- if cmpCN == 0: # dueling insert
- raise merge_error(6)
- if cmpCN > 0: # insert new
- merge_output(i_new)
- else: # insert committed
- merge_output(i_com)
-
- while i_old.active and i_com.active: # new deletes rest of original
- cmpOC = compare(i_old.key, i_com.key)
- if cmpOC > 0: # insert committed
- merge_output(i_com)
- elif cmpOC == 0: # del in new
- i_old.advance()
- i_com.advance()
- else: # dueling deletes or delete and change
- raise merge_error(7)
-
- while i_old.active and i_new.active:
- # committed deletes rest of original
- cmpON = compare(i_old.key, i_new.key)
- if cmpON > 0: # insert new
- merge_output(i_new)
- elif cmpON == 0: # deleted in committed
- i_old.advance()
- i_new.advance()
- else: # dueling deletes or delete and change
- raise merge_error(8)
-
- if i_old.active: # dueling deletes
- raise merge_error(9)
-
- while i_com.active:
- merge_output(i_com)
-
- while i_new.active:
- merge_output(i_new)
-
- if len(result._keys) == 0: #pragma: no cover
- # If the output bucket is empty, conflict resolution doesn't have
- # enough info to unlink it from its containing BTree correctly.
- #
- # XXX TS, 2012-11-16: I don't think this is possible
- #
- raise merge_error(10)
-
- result._next = b_old._next
- return result.__getstate__()
-
- def __repr__(self):
- return self._repr_helper(self._keys)
-
- class _TreeItem(object):
-
- __slots__ = ('key',
- 'child',
- )
-
- def __init__(self, key, child):
- self.key = key
- self.child = child
-
-
- class _Tree(_Base):
-
- __slots__ = ('_data',
- '_firstbucket',
- )
-
- def __new__(cls, *args):
- value = _Base.__new__(cls, *args)
- # Empty trees don't get their __setstate__ called upon
- # unpickling (or __init__, obviously), so clear() is never called
- # and _data and _firstbucket are never defined, unless we do it here.
- value._data = []
- value._firstbucket = None
- return value
-
- def setdefault(self, key, value):
- return self._set(self._to_key(key), self._to_value(value), True)[1]
-
- def pop(self, key, default=_marker):
- try:
- return self._del(self._to_key(key))[1]
- except KeyError:
- if default is _marker:
- raise
- return default
-
- def update(self, items):
- if hasattr(items, 'iteritems'):
- items = items.iteritems()
- elif hasattr(items, 'items'):
- items = items.items()
-
- set = self.__setitem__
- for i in items:
- set(*i)
-
- def __setitem__(self, key, value):
- _no_default_comparison(key)
- self._set(self._to_key(key), self._to_value(value))
-
- def __delitem__(self, key):
- self._del(self._to_key(key))
-
- def clear(self):
- if self._data:
- # In the case of __init__, this was already set by __new__
- self._data = []
- self._firstbucket = None
-
- def __nonzero__(self):
- return bool(self._data)
- __bool__ = __nonzero__ #Py3k rename
-
- def __len__(self):
- l = 0
- bucket = self._firstbucket
- while bucket is not None:
- l += len(bucket._keys)
- bucket = bucket._next
- return l
-
- @property
- def size(self):
- return len(self._data)
-
- def _search(self, key):
- data = self._data
- if data:
- lo = 0
- hi = len(data)
- i = hi // 2
- while i > lo:
- cmp_ = compare(data[i].key, key)
- if cmp_ < 0:
- lo = i
- elif cmp_ > 0:
- hi = i
- else:
- break
- i = (lo + hi) // 2
- return i
- return -1
-
- def _findbucket(self, key):
- index = self._search(key)
- if index >= 0:
- child = self._data[index].child
- if isinstance(child, self._bucket_type):
- return child
- return child._findbucket(key)
-
- def __contains__(self, key):
- try:
- tree_key = self._to_key(key)
- except TypeError:
- # Can't convert the key, so can't possibly be in the tree
- return False
- return key in (self._findbucket(tree_key) or ())
-
- def has_key(self, key):
- index = self._search(key)
- if index < 0:
- return False
- r = self._data[index].child.has_key(key)
- return r and r + 1
-
- def keys(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False,
- itertype='iterkeys'):
- if not self._data:
- return ()
-
- if min is not _marker and min is not None:
- min = self._to_key(min)
- bucket = self._findbucket(min)
- else:
- bucket = self._firstbucket
-
- iterargs = min, max, excludemin, excludemax
-
- return _TreeItems(bucket, itertype, iterargs)
-
- def iterkeys(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- return iter(self.keys(min, max, excludemin, excludemax))
-
- def __iter__(self):
- return iter(self.keys())
-
- def minKey(self, min=_marker):
- if min is _marker or min is None:
- bucket = self._firstbucket
- else:
- min = self._to_key(min)
- bucket = self._findbucket(min)
- if bucket is not None:
- return bucket.minKey(min)
- raise ValueError('empty tree')
-
- def maxKey(self, max=_marker):
- data = self._data
- if not data:
- raise ValueError('empty tree')
- if max is _marker or max is None:
- return data[-1].child.maxKey()
-
- max = self._to_key(max)
- index = self._search(max)
- if index and compare(data[index].child.minKey(), max) > 0:
- index -= 1 #pragma: no cover no idea how to provoke this
- return data[index].child.maxKey(max)
-
-
- def _set(self, key, value=None, ifunset=False):
- if (self._p_jar is not None and
- self._p_oid is not None and
- self._p_serial is not None):
- self._p_jar.readCurrent(self)
- data = self._data
- if data:
- index = self._search(key)
- child = data[index].child
- else:
- index = 0
- child = self._bucket_type()
- self._firstbucket = child
- data.append(_TreeItem(None, child))
-
- result = child._set(key, value, ifunset)
- grew = result[0]
- if grew:
- if type(child) is type(self):
- max_size = self.max_internal_size
- else:
- max_size = self.max_leaf_size
- if child.size > max_size:
- self._grow(child, index)
-
- # If a BTree contains only a single bucket, BTree.__getstate__()
- # includes the bucket's entire state, and the bucket doesn't get
- # an oid of its own. So if we have a single oid-less bucket that
- # changed, it's *our* oid that should be marked as changed -- the
- # bucket doesn't have one.
- if (grew is not None and
- type(child) is self._bucket_type and
- len(data) == 1 and
- child._p_oid is None):
- self._p_changed = 1
- return result
-
- def _grow(self, child, index):
- self._p_changed = True
- new_child = child._split()
- self._data.insert(index+1, _TreeItem(new_child.minKey(), new_child))
- if len(self._data) >= self.max_internal_size * 2:
- self._split_root()
-
- def _split_root(self):
- child = type(self)()
- child._data = self._data
- child._firstbucket = self._firstbucket
- self._data = [_TreeItem(None, child)]
- self._grow(child, 0)
-
- def _split(self, index=None):
- data = self._data
- if index is None:
- index = len(data) // 2
-
- next = type(self)()
- next._data = data[index:]
- first = data[index]
- del data[index:]
- if len(data) == 0:
- self._firstbucket = None # lost our bucket, can't buy no beer
- if isinstance(first.child, type(self)):
- next._firstbucket = first.child._firstbucket
- else:
- next._firstbucket = first.child;
- return next
-
- def _del(self, key):
- if (self._p_jar is not None and
- self._p_oid is not None and
- self._p_serial is not None):
- self._p_jar.readCurrent(self)
-
- data = self._data
- if not data:
- raise KeyError(key)
-
- index = self._search(key)
- child = data[index].child
-
- removed_first_bucket, value = child._del(key)
-
- # See comment in _set about small trees
- if (len(data) == 1 and
- type(child) is self._bucket_type and
- child._p_oid is None):
- self._p_changed = True
-
- # fix up the node key, but not for the 0'th one.
- if index > 0 and child.size and compare(key, data[index].key) == 0:
- self._p_changed = True
- data[index].key = child.minKey()
-
- if removed_first_bucket:
- if index:
- data[index-1].child._deleteNextBucket()
- removed_first_bucket = False # clear flag
- else:
- self._firstbucket = child._firstbucket
-
- if not child.size:
- if type(child) is self._bucket_type:
- if index:
- data[index-1].child._deleteNextBucket()
- else:
- self._firstbucket = child._next
- removed_first_bucket = True
- del data[index]
-
- return removed_first_bucket, value
-
- def _deleteNextBucket(self):
- self._data[-1].child._deleteNextBucket()
-
- def __getstate__(self):
- data = self._data
-
- if not data:
- # Note: returning None here causes our __setstate__
- # to not be called on unpickling
- return None
-
- if (len(data) == 1 and
- type(data[0].child) is not type(self) and
- data[0].child._p_oid is None
- ):
- return ((data[0].child.__getstate__(), ), )
-
- sdata = []
- for item in data:
- if sdata:
- sdata.append(item.key)
- sdata.append(item.child)
- else:
- sdata.append(item.child)
-
- return tuple(sdata), self._firstbucket
-
- def __setstate__(self, state):
- if state and not isinstance(state[0], tuple):
- raise TypeError('tuple required for first state element')
-
- self.clear()
- if state is None:
- return
-
- if len(state) == 1:
- bucket = self._bucket_type()
- bucket.__setstate__(state[0][0])
- state = [bucket], bucket
-
- data, self._firstbucket = state
- data = list(reversed(data))
-
- self._data.append(_TreeItem(None, data.pop()))
- while data:
- key = data.pop()
- child = data.pop()
- self._data.append(_TreeItem(key, child))
-
- def _assert(self, condition, message):
- if not condition:
- raise AssertionError(message)
-
- def _check(self, nextbucket=None):
- data = self._data
- assert_ = self._assert
- if not data:
- assert_(self._firstbucket is None,
- "Empty BTree has non-NULL firstbucket")
- return
- assert_(self._firstbucket is not None,
- "Non-empty BTree has NULL firstbucket")
-
- child_class = type(data[0].child)
- for i in data:
- assert_(i.child is not None, "BTree has NULL child")
- assert_(type(i.child) is child_class,
- "BTree children have different types")
- assert_(i.child.size, "Bucket length < 1")
-
- if child_class is type(self):
- assert_(self._firstbucket is data[0].child._firstbucket,
- "BTree has firstbucket different than "
- "its first child's firstbucket")
- for i in range(len(data)-1):
- data[i].child._check(data[i+1].child._firstbucket)
- data[-1].child._check(nextbucket)
- elif child_class is self._bucket_type:
- assert_(self._firstbucket is data[0].child,
- "Bottom-level BTree node has inconsistent firstbucket "
- "belief")
- for i in range(len(data)-1):
- assert_(data[i].child._next is data[i+1].child,
- "Bucket next pointer is damaged")
- assert_(data[-1].child._next is nextbucket,
- "Bucket next pointer is damaged")
- else:
- assert_(False, "Incorrect child type")
-
- def _p_resolveConflict(self, old, com, new):
- s_old = _get_simple_btree_bucket_state(old)
- s_com = _get_simple_btree_bucket_state(com)
- s_new = _get_simple_btree_bucket_state(new)
- return ((
- self._bucket_type()._p_resolveConflict(s_old, s_com, s_new), ), )
-
- def __repr__(self):
- r = super(_Tree, self).__repr__()
- r = r.replace('Py', '')
- return r
-
- def _get_simple_btree_bucket_state(state):
- if state is None:
- return state
- if not isinstance(state, tuple):
- raise TypeError("_p_resolveConflict: expected tuple or None for state")
- if len(state) == 2: # non-degenerate BTree, can't resolve
- raise BTreesConflictError(-1, -1, -1, 11)
- # Peel away wrapper to get to only-bucket state.
- if len(state) != 1:
- raise TypeError("_p_resolveConflict: expected 1- or 2-tuple for state")
- state = state[0]
- if not isinstance(state, tuple) or len(state) != 1:
- raise TypeError("_p_resolveConflict: expected 1-tuple containing "
- "bucket state")
- state = state[0]
- if not isinstance(state, tuple):
- raise TypeError("_p_resolveConflict: expected tuple for bucket state")
- return state
-
-
- class _TreeItems(object):
-
- __slots__ = ('firstbucket',
- 'itertype',
- 'iterargs',
- 'index',
- 'it',
- 'v',
- '_len',
- )
-
- def __init__(self, firstbucket, itertype, iterargs):
- self.firstbucket = firstbucket
- self.itertype = itertype
- self.iterargs = iterargs
- self.index = -1
- self.it = iter(self)
- self.v = None
- self._len = None
-
- def __getitem__(self, i):
- if isinstance(i, slice):
- return list(self)[i]
- if i < 0:
- i = len(self) + i
- if i < 0:
- raise IndexError(i)
-
- if i < self.index:
- self.index = -1
- self.it = iter(self)
-
- while i > self.index:
- try:
- self.v = next(self.it)
- except StopIteration:
- raise IndexError(i)
- else:
- self.index += 1
- return self.v
-
- def __len__(self):
- if self._len is None:
- i = 0
- for _ in self:
- i += 1
- self._len = i
- return self._len
-
- def __iter__(self):
- bucket = self.firstbucket
- itertype = self.itertype
- iterargs = self.iterargs
- done = 0
- # Note that we don't mind if the first bucket yields no
- # results due to an idiosyncrasy in how range searches are done.
- while bucket is not None:
- for k in getattr(bucket, itertype)(*iterargs):
- yield k
- done = 0
- if done:
- return
- bucket = bucket._next
- done = 1
-
-
- class _TreeIterator(object):
- """ Faux implementation for BBB only.
- """
- def __init__(self, items): #pragma: no cover
- raise TypeError(
- "TreeIterators are private implementation details "
- "of the C-based BTrees.\n\n"
- "Please use 'iter(tree)', rather than instantiating "
- "one directly."
- )
-
-
- class Tree(_Tree):
-
- __slots__ = ()
-
- def get(self, key, default=None):
- bucket = self._findbucket(key)
- if bucket:
- return bucket.get(key, default)
- return default
-
- def __getitem__(self, key):
- bucket = self._findbucket(key)
- if bucket:
- return bucket[key]
- raise KeyError(key)
-
- def values(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- return self.keys(min, max, excludemin, excludemax, 'itervalues')
-
- def itervalues(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- return iter(self.values(min, max, excludemin, excludemax))
-
- def items(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- return self.keys(min, max, excludemin, excludemax, 'iteritems')
-
- def iteritems(self, min=_marker, max=_marker,
- excludemin=False, excludemax=False):
- return iter(self.items(min, max, excludemin, excludemax))
-
- def byValue(self, min):
- return reversed(
- sorted((v, k) for (k, v) in self.iteritems() if v >= min))
-
- def insert(self, key, value):
- return bool(self._set(key, value, True)[0])
-
-
- class TreeSet(_Tree):
-
- __slots__ = ()
-
- def add(self, key):
- return self._set(self._to_key(key))[0]
-
- insert = add
-
- def remove(self, key):
- self._del(self._to_key(key))
-
- def update(self, items):
- add = self.add
- for i in items:
- add(i)
-
- _p_resolveConflict = _Tree._p_resolveConflict
-
-
- class set_operation(object):
-
- __slots__ = ('func',
- 'set_type',
- )
-
- def __init__(self, func, set_type):
- self.func = func
- self.set_type = set_type
-
- def __call__(self, *a, **k):
- return self.func(self.set_type, *a, **k)
-
-
- def difference(set_type, o1, o2):
- if o1 is None or o2 is None:
- return o1
- i1 = _SetIteration(o1, True, 0)
- i2 = _SetIteration(o2, False, 0)
- if i1.useValues:
- result = o1._mapping_type()
- def copy(i):
- result._keys.append(i.key)
- result._values.append(i.value)
- else:
- result = o1._set_type()
- def copy(i):
- result._keys.append(i.key)
- while i1.active and i2.active:
- cmp_ = compare(i1.key, i2.key)
- if cmp_ < 0:
- copy(i1)
- i1.advance()
- elif cmp_ == 0:
- i1.advance()
- i2.advance()
- else:
- i2.advance()
- while i1.active:
- copy(i1)
- i1.advance()
- return result
-
- def union(set_type, o1, o2):
- if o1 is None:
- return o2
- if o2 is None:
- return o1
- i1 = _SetIteration(o1, False, 0)
- i2 = _SetIteration(o2, False, 0)
- result = o1._set_type()
- def copy(i):
- result._keys.append(i.key)
- while i1.active and i2.active:
- cmp_ = compare(i1.key, i2.key)
- if cmp_ < 0:
- copy(i1)
- i1.advance()
- elif cmp_ == 0:
- copy(i1)
- i1.advance()
- i2.advance()
- else:
- copy(i2)
- i2.advance()
- while i1.active:
- copy(i1)
- i1.advance()
- while i2.active:
- copy(i2)
- i2.advance()
- return result
-
- def intersection(set_type, o1, o2):
- if o1 is None:
- return o2
- if o2 is None:
- return o1
- i1 = _SetIteration(o1, False, 0)
- i2 = _SetIteration(o2, False, 0)
- result = o1._set_type()
- def copy(i):
- result._keys.append(i.key)
- while i1.active and i2.active:
- cmp_ = compare(i1.key, i2.key)
- if cmp_ < 0:
- i1.advance()
- elif cmp_ == 0:
- copy(i1)
- i1.advance()
- i2.advance()
- else:
- i2.advance()
- return result
-
- def _prepMergeIterators(o1, o2):
- MERGE_DEFAULT = getattr(o1, 'MERGE_DEFAULT', None)
- if MERGE_DEFAULT is None:
- raise TypeError("invalid set operation")
- i1 = _SetIteration(o1, True, MERGE_DEFAULT)
- i2 = _SetIteration(o2, True, MERGE_DEFAULT)
- return i1, i2
-
- def weightedUnion(set_type, o1, o2, w1=1, w2=1):
- if o1 is None:
- if o2 is None:
- return 0, None
- return w2, o2
- if o2 is None:
- return w1, o1
- i1, i2 = _prepMergeIterators(o1, o2)
- MERGE = getattr(o1, 'MERGE', None)
- if MERGE is None and i1.useValues and i2.useValues:
- raise TypeError("invalid set operation")
- MERGE_WEIGHT = getattr(o1, 'MERGE_WEIGHT')
- if (not i1.useValues) and i2.useValues:
- i1, i2 = i2, i1
- w1, w2 = w2, w1
- _merging = i1.useValues or i2.useValues
- if _merging:
- result = o1._mapping_type()
- def copy(i, w):
- result._keys.append(i.key)
- result._values.append(MERGE_WEIGHT(i.value, w))
- else:
- result = o1._set_type()
- def copy(i, w):
- result._keys.append(i.key)
-
- while i1.active and i2.active:
- cmp_ = compare(i1.key, i2.key)
- if cmp_ < 0:
- copy(i1, w1)
- i1.advance()
- elif cmp_ == 0:
- result._keys.append(i1.key)
- if _merging:
- result._values.append(MERGE(i1.value, w1, i2.value, w2))
- i1.advance()
- i2.advance()
- else:
- copy(i2, w2)
- i2.advance()
- while i1.active:
- copy(i1, w1)
- i1.advance()
- while i2.active:
- copy(i2, w2)
- i2.advance()
- return 1, result
-
- def weightedIntersection(set_type, o1, o2, w1=1, w2=1):
- if o1 is None:
- if o2 is None:
- return 0, None
- return w2, o2
- if o2 is None:
- return w1, o1
- i1, i2 = _prepMergeIterators(o1, o2)
- MERGE = getattr(o1, 'MERGE', None)
- if MERGE is None and i1.useValues and i2.useValues:
- raise TypeError("invalid set operation")
- if (not i1.useValues) and i2.useValues:
- i1, i2 = i2, i1
- w1, w2 = w2, w1
- _merging = i1.useValues or i2.useValues
- if _merging:
- result = o1._mapping_type()
- else:
- result = o1._set_type()
- while i1.active and i2.active:
- cmp_ = compare(i1.key, i2.key)
- if cmp_ < 0:
- i1.advance()
- elif cmp_ == 0:
- result._keys.append(i1.key)
- if _merging:
- result._values.append(MERGE(i1.value, w1, i2.value, w2))
- i1.advance()
- i2.advance()
- else:
- i2.advance()
- if isinstance(result, (Set, TreeSet)):
- return w1 + w2, result
- return 1, result
-
- def multiunion(set_type, seqs):
- # XXX simple/slow implementation. Goal is just to get tests to pass.
- result = set_type()
- for s in seqs:
- try:
- iter(s)
- except TypeError:
- s = set_type((s, ))
- result.update(s)
- return result
-
- def to_ob(self, v):
- return v
-
- def _packer_unpacker(struct_format):
- s = Struct(struct_format)
- return s.pack, s.unpack
-
- int_pack, int_unpack = _packer_unpacker('i')
-
- def to_int(self, v):
- try:
- int_pack(index(v))
- except (struct_error, TypeError):
- raise TypeError('32-bit integer expected')
-
- return int(v)
-
- float_pack = _packer_unpacker('f')[0]
-
- def to_float(self, v):
- try:
- float_pack(v)
- except struct_error:
- raise TypeError('float expected')
- return float(v)
-
-
- long_pack, long_unpack = _packer_unpacker('q')
-
- def to_long(self, v):
- try:
- long_pack(index(v))
- except (struct_error, TypeError):
- if isinstance(v, int_types):
- raise ValueError("Value out of range", v)
- raise TypeError('64-bit integer expected')
-
- return int(v)
-
- def to_bytes(l):
- def to(self, v):
- if not (isinstance(v, bytes) and len(v) == l):
- raise TypeError("%s-byte array expected" % l)
- return v
- return to
-
- tos = dict(I=to_int, L=to_long, F=to_float, O=to_ob)
-
- MERGE_DEFAULT_int = 1
- MERGE_DEFAULT_float = 1.0
-
- def MERGE(self, value1, weight1, value2, weight2):
- return (value1 * weight1) + (value2 * weight2)
-
- def MERGE_WEIGHT_default(self, value, weight):
- return value
-
- def MERGE_WEIGHT_numeric(self, value, weight):
- return value * weight
-
- def _fix_pickle(mod_dict, mod_name):
- # Make the pure-Python objects pickle with the same
- # class names and types as the C extensions by setting the appropriate
- # _BTree_reduce_as attribute.
- # If the C extensions are not available, we also change the
- # __name__ attribute of the type to match the C name (otherwise
- # we wind up with *Py in the pickles)
- # Each module must call this as `_fix_pickle(globals(), __name__)`
- # at the bottom.
-
- mod_prefix = mod_name.split('.')[-1][:2] # BTrees.OOBTree -> 'OO'
- bucket_name = mod_prefix + 'Bucket'
- py_bucket_name = bucket_name + 'Py'
-
- have_c_extensions = mod_dict[bucket_name] is not mod_dict[py_bucket_name]
-
- for name in 'Bucket', 'Set', 'BTree', 'TreeSet', 'TreeIterator':
- raw_name = mod_prefix + name
- py_name = raw_name + 'Py'
- try:
- py_type = mod_dict[py_name]
- except KeyError:
- if name == 'TreeIterator':
- # Optional
- continue
- raise # pragma: no cover
- raw_type = mod_dict[raw_name] # Could be C or Python
-
- py_type._BTree_reduce_as = raw_type
- py_type._BTree_reduce_up_bound = py_type
-
- if not have_c_extensions: # pragma: no cover
- # Set FooPy to have the __name__ of simply Foo.
- # We can't do this if the C extension is available,
- # because then mod_dict[FooPy.__name__] is not FooPy
- # and pickle refuses to save something like that.
- # On the other hand (no C extension) this makes our
- # Python pickle match the C version by default
- py_type.__name__ = raw_name
- py_type.__qualname__ = raw_name # Py 3.3+
|