tested django-newsletter
This commit is contained in:
Esther Kleinhenz
@@ -0,0 +1,109 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""ZODB transfer activity monitoring
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from . import utils
|
||||
|
||||
|
||||
class ActivityMonitor(object):
|
||||
"""ZODB load/store activity monitor
|
||||
|
||||
This simple implementation just keeps a small log in memory
|
||||
and iterates over the log when getActivityAnalysis() is called.
|
||||
|
||||
It assumes that log entries are added in chronological sequence.
|
||||
"""
|
||||
|
||||
def __init__(self, history_length=3600):
|
||||
self.history_length = history_length # Number of seconds
|
||||
self.log = [] # [(time, loads, stores)]
|
||||
self.trim_lock = utils.Lock()
|
||||
|
||||
def closedConnection(self, conn):
|
||||
log = self.log
|
||||
now = time.time()
|
||||
loads, stores = conn.getTransferCounts(1)
|
||||
log.append((now, loads, stores))
|
||||
self.trim(now)
|
||||
|
||||
def trim(self, now):
|
||||
with self.trim_lock:
|
||||
log = self.log
|
||||
cutoff = now - self.history_length
|
||||
n = 0
|
||||
loglen = len(log)
|
||||
while n < loglen and log[n][0] < cutoff:
|
||||
n = n + 1
|
||||
if n:
|
||||
del log[:n]
|
||||
|
||||
def setHistoryLength(self, history_length):
|
||||
self.history_length = history_length
|
||||
self.trim(time.time())
|
||||
|
||||
def getHistoryLength(self):
|
||||
return self.history_length
|
||||
|
||||
def getActivityAnalysis(self, start=0, end=0, divisions=10):
|
||||
res = []
|
||||
now = time.time()
|
||||
if start == 0:
|
||||
start = now - self.history_length
|
||||
if end == 0:
|
||||
end = now
|
||||
for n in range(divisions):
|
||||
res.append({
|
||||
'start': start + (end - start) * n / divisions,
|
||||
'end': start + (end - start) * (n + 1) / divisions,
|
||||
'loads': 0,
|
||||
'stores': 0,
|
||||
'connections': 0,
|
||||
})
|
||||
|
||||
div = res[0]
|
||||
div_end = div['end']
|
||||
div_index = 0
|
||||
connections = 0
|
||||
total_loads = 0
|
||||
total_stores = 0
|
||||
for t, loads, stores in self.log:
|
||||
if t < start:
|
||||
# We could use a binary search to find the start.
|
||||
continue
|
||||
elif t > end:
|
||||
# We could use a binary search to find the end also.
|
||||
break
|
||||
while t > div_end:
|
||||
div['loads'] = total_loads
|
||||
div['stores'] = total_stores
|
||||
div['connections'] = connections
|
||||
total_loads = 0
|
||||
total_stores = 0
|
||||
connections = 0
|
||||
div_index = div_index + 1
|
||||
if div_index < divisions:
|
||||
div = res[div_index]
|
||||
div_end = div['end']
|
||||
connections = connections + 1
|
||||
total_loads = total_loads + loads
|
||||
total_stores = total_stores + stores
|
||||
|
||||
div['stores'] = div['stores'] + total_stores
|
||||
div['loads'] = div['loads'] + total_loads
|
||||
div['connections'] = div['connections'] + connections
|
||||
|
||||
return res
|
||||
@@ -0,0 +1,381 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Storage base class that is mostly a mistake
|
||||
|
||||
The base class here is tightly coupled with its subclasses and
|
||||
its use is not recommended. It's still here for historical reasons.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import logging
|
||||
import sys
|
||||
from struct import pack as _structpack, unpack as _structunpack
|
||||
|
||||
import zope.interface
|
||||
from persistent.TimeStamp import TimeStamp
|
||||
|
||||
import ZODB.interfaces
|
||||
from . import POSException, utils
|
||||
from .Connection import TransactionMetaData
|
||||
from .utils import z64, oid_repr, byte_ord, byte_chr, load_current
|
||||
from .UndoLogCompatible import UndoLogCompatible
|
||||
from ._compat import dumps, _protocol, py2_hasattr
|
||||
|
||||
log = logging.getLogger("ZODB.BaseStorage")
|
||||
|
||||
class BaseStorage(UndoLogCompatible):
|
||||
"""Base class that supports storage implementations.
|
||||
|
||||
XXX Base classes like this are an attractive nuisance. They often
|
||||
introduce more complexity than they save. While important logic
|
||||
is implemented here, we should consider exposing it as utility
|
||||
functions or as objects that can be used through composition.
|
||||
|
||||
A subclass must define the following methods:
|
||||
load()
|
||||
store()
|
||||
close()
|
||||
cleanup()
|
||||
lastTransaction()
|
||||
|
||||
It must override these hooks:
|
||||
_begin()
|
||||
_vote()
|
||||
_abort()
|
||||
_finish()
|
||||
_clear_temp()
|
||||
|
||||
If it stores multiple revisions, it should implement
|
||||
loadSerial()
|
||||
loadBefore()
|
||||
|
||||
Each storage will have two locks that are accessed via lock
|
||||
acquire and release methods bound to the instance. (Yuck.)
|
||||
_lock_acquire / _lock_release (reentrant)
|
||||
_commit_lock_acquire / _commit_lock_release
|
||||
|
||||
The commit lock is acquired in tpc_begin() and released in
|
||||
tpc_abort() and tpc_finish(). It is never acquired with the other
|
||||
lock held.
|
||||
|
||||
The other lock appears to protect _oid and _transaction and
|
||||
perhaps other things. It is always held when load() is called, so
|
||||
presumably the load() implementation should also acquire the lock.
|
||||
"""
|
||||
_transaction=None # Transaction that is being committed
|
||||
_tstatus=' ' # Transaction status, used for copying data
|
||||
_is_read_only = False
|
||||
|
||||
def __init__(self, name, base=None):
|
||||
self.__name__= name
|
||||
log.debug("create storage %s", self.__name__)
|
||||
|
||||
# Allocate locks:
|
||||
self._lock = utils.RLock()
|
||||
self._commit_lock = utils.Lock()
|
||||
|
||||
# Needed by external storages that use this dumb api :(
|
||||
self._lock_acquire = self._lock.acquire
|
||||
self._lock_release = self._lock.release
|
||||
self._commit_lock_acquire = self._commit_lock.acquire
|
||||
self._commit_lock_release = self._commit_lock.release
|
||||
|
||||
t = time.time()
|
||||
t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t%60,)))
|
||||
self._tid = t.raw()
|
||||
|
||||
# ._oid is the highest oid in use (0 is always in use -- it's
|
||||
# a reserved oid for the root object). Our new_oid() method
|
||||
# increments it by 1, and returns the result. It's really a
|
||||
# 64-bit integer stored as an 8-byte big-endian string.
|
||||
oid = getattr(base, '_oid', None)
|
||||
if oid is None:
|
||||
self._oid = z64
|
||||
else:
|
||||
self._oid = oid
|
||||
# In case that conflicts are resolved during store,
|
||||
# this collects oids to be returned by tpc_vote.
|
||||
self._resolved = []
|
||||
|
||||
def sortKey(self):
|
||||
"""Return a string that can be used to sort storage instances.
|
||||
|
||||
The key must uniquely identify a storage and must be the same
|
||||
across multiple instantiations of the same storage.
|
||||
"""
|
||||
# name may not be sufficient, e.g. ZEO has a user-definable name.
|
||||
return self.__name__
|
||||
|
||||
def getName(self):
|
||||
return self.__name__
|
||||
|
||||
def getSize(self):
|
||||
return len(self)*300 # WAG!
|
||||
|
||||
def history(self, oid, version, length=1, filter=None):
|
||||
return ()
|
||||
|
||||
def new_oid(self):
|
||||
if self._is_read_only:
|
||||
raise POSException.ReadOnlyError()
|
||||
|
||||
with self._lock:
|
||||
last = self._oid
|
||||
d = byte_ord(last[-1])
|
||||
if d < 255: # fast path for the usual case
|
||||
last = last[:-1] + byte_chr(d+1)
|
||||
else: # there's a carry out of the last byte
|
||||
last_as_long, = _structunpack(">Q", last)
|
||||
last = _structpack(">Q", last_as_long + 1)
|
||||
self._oid = last
|
||||
return last
|
||||
|
||||
# Update the maximum oid in use, under protection of a lock. The
|
||||
# maximum-in-use attribute is changed only if possible_new_max_oid is
|
||||
# larger than its current value.
|
||||
def set_max_oid(self, possible_new_max_oid):
|
||||
with self._lock:
|
||||
if possible_new_max_oid > self._oid:
|
||||
self._oid = possible_new_max_oid
|
||||
|
||||
def registerDB(self, db):
|
||||
pass # we don't care
|
||||
|
||||
def isReadOnly(self):
|
||||
return self._is_read_only
|
||||
|
||||
def tpc_abort(self, transaction):
|
||||
with self._lock:
|
||||
|
||||
if transaction is not self._transaction:
|
||||
return
|
||||
|
||||
try:
|
||||
self._abort()
|
||||
self._clear_temp()
|
||||
self._transaction = None
|
||||
finally:
|
||||
self._commit_lock_release()
|
||||
|
||||
def _abort(self):
|
||||
"""Subclasses should redefine this to supply abort actions"""
|
||||
pass
|
||||
|
||||
def tpc_begin(self, transaction, tid=None, status=' '):
|
||||
if self._is_read_only:
|
||||
raise POSException.ReadOnlyError()
|
||||
|
||||
with self._lock:
|
||||
if self._transaction is transaction:
|
||||
raise POSException.StorageTransactionError(
|
||||
"Duplicate tpc_begin calls for same transaction")
|
||||
|
||||
self._commit_lock.acquire()
|
||||
|
||||
with self._lock:
|
||||
self._transaction = transaction
|
||||
self._clear_temp()
|
||||
|
||||
user = transaction.user
|
||||
desc = transaction.description
|
||||
ext = transaction.extension
|
||||
if ext:
|
||||
ext = dumps(ext, _protocol)
|
||||
else:
|
||||
ext = ""
|
||||
|
||||
self._ude = user, desc, ext
|
||||
|
||||
if tid is None:
|
||||
now = time.time()
|
||||
t = TimeStamp(*(time.gmtime(now)[:5] + (now % 60,)))
|
||||
self._ts = t = t.laterThan(self._ts)
|
||||
self._tid = t.raw()
|
||||
else:
|
||||
self._ts = TimeStamp(tid)
|
||||
self._tid = tid
|
||||
|
||||
del self._resolved[:]
|
||||
self._tstatus = status
|
||||
self._begin(self._tid, user, desc, ext)
|
||||
|
||||
def tpc_transaction(self):
|
||||
return self._transaction
|
||||
|
||||
def _begin(self, tid, u, d, e):
|
||||
"""Subclasses should redefine this to supply transaction start actions.
|
||||
"""
|
||||
pass
|
||||
|
||||
def tpc_vote(self, transaction):
|
||||
with self._lock:
|
||||
if transaction is not self._transaction:
|
||||
raise POSException.StorageTransactionError(
|
||||
"tpc_vote called with wrong transaction")
|
||||
return self._vote()
|
||||
|
||||
def _vote(self):
|
||||
"""Subclasses should redefine this to supply transaction vote actions.
|
||||
"""
|
||||
return self._resolved
|
||||
|
||||
def tpc_finish(self, transaction, f=None):
|
||||
# It's important that the storage calls the function we pass
|
||||
# while it still has its lock. We don't want another thread
|
||||
# to be able to read any updated data until we've had a chance
|
||||
# to send an invalidation message to all of the other
|
||||
# connections!
|
||||
|
||||
with self._lock:
|
||||
if transaction is not self._transaction:
|
||||
raise POSException.StorageTransactionError(
|
||||
"tpc_finish called with wrong transaction")
|
||||
try:
|
||||
if f is not None:
|
||||
f(self._tid)
|
||||
u, d, e = self._ude
|
||||
self._finish(self._tid, u, d, e)
|
||||
self._clear_temp()
|
||||
finally:
|
||||
self._ude = None
|
||||
self._transaction = None
|
||||
self._commit_lock.release()
|
||||
return self._tid
|
||||
|
||||
def _finish(self, tid, u, d, e):
|
||||
"""Subclasses should redefine this to supply transaction finish actions
|
||||
"""
|
||||
pass
|
||||
|
||||
def lastTransaction(self):
|
||||
with self._lock:
|
||||
return self._ltid
|
||||
|
||||
def getTid(self, oid):
|
||||
with self._lock:
|
||||
return load_current(self, oid)[1]
|
||||
|
||||
def loadSerial(self, oid, serial):
|
||||
raise POSException.Unsupported(
|
||||
"Retrieval of historical revisions is not supported")
|
||||
|
||||
def loadBefore(self, oid, tid):
|
||||
"""Return most recent revision of oid before tid committed."""
|
||||
return None
|
||||
|
||||
def copyTransactionsFrom(self, other, verbose=0):
|
||||
"""Copy transactions from another storage.
|
||||
|
||||
This is typically used for converting data from one storage to
|
||||
another. `other` must have an .iterator() method.
|
||||
"""
|
||||
copy(other, self, verbose)
|
||||
|
||||
def copy(source, dest, verbose=0):
|
||||
"""Copy transactions from a source to a destination storage
|
||||
|
||||
This is typically used for converting data from one storage to
|
||||
another. `source` must have an .iterator() method.
|
||||
"""
|
||||
_ts = None
|
||||
ok = 1
|
||||
preindex = {};
|
||||
preget = preindex.get
|
||||
# restore() is a new storage API method which has an identical
|
||||
# signature to store() except that it does not return anything.
|
||||
# Semantically, restore() is also identical to store() except that it
|
||||
# doesn't do the ConflictError or VersionLockError consistency
|
||||
# checks. The reason to use restore() over store() in this method is
|
||||
# that store() cannot be used to copy transactions spanning a version
|
||||
# commit or abort, or over transactional undos.
|
||||
#
|
||||
# We'll use restore() if it's available, otherwise we'll fall back to
|
||||
# using store(). However, if we use store, then
|
||||
# copyTransactionsFrom() may fail with VersionLockError or
|
||||
# ConflictError.
|
||||
restoring = py2_hasattr(dest, 'restore')
|
||||
fiter = source.iterator()
|
||||
for transaction in fiter:
|
||||
tid = transaction.tid
|
||||
if _ts is None:
|
||||
_ts = TimeStamp(tid)
|
||||
else:
|
||||
t = TimeStamp(tid)
|
||||
if t <= _ts:
|
||||
if ok: print(('Time stamps out of order %s, %s' % (_ts, t)))
|
||||
ok = 0
|
||||
_ts = t.laterThan(_ts)
|
||||
tid = _ts.raw()
|
||||
else:
|
||||
_ts = t
|
||||
if not ok:
|
||||
print(('Time stamps back in order %s' % (t)))
|
||||
ok = 1
|
||||
|
||||
if verbose:
|
||||
print(_ts)
|
||||
|
||||
dest.tpc_begin(transaction, tid, transaction.status)
|
||||
for r in transaction:
|
||||
oid = r.oid
|
||||
if verbose:
|
||||
print(oid_repr(oid), r.version, len(r.data))
|
||||
if restoring:
|
||||
dest.restore(oid, r.tid, r.data, r.version,
|
||||
r.data_txn, transaction)
|
||||
else:
|
||||
pre = preget(oid, None)
|
||||
dest.store(oid, pre, r.data, r.version, transaction)
|
||||
preindex[oid] = tid
|
||||
|
||||
dest.tpc_vote(transaction)
|
||||
dest.tpc_finish(transaction)
|
||||
|
||||
|
||||
# defined outside of BaseStorage to facilitate independent reuse.
|
||||
# just depends on _transaction attr and getTid method.
|
||||
def checkCurrentSerialInTransaction(self, oid, serial, transaction):
|
||||
if transaction is not self._transaction:
|
||||
raise POSException.StorageTransactionError(self, transaction)
|
||||
|
||||
committed_tid = self.getTid(oid)
|
||||
if committed_tid != serial:
|
||||
raise POSException.ReadConflictError(
|
||||
oid=oid, serials=(committed_tid, serial))
|
||||
|
||||
BaseStorage.checkCurrentSerialInTransaction = checkCurrentSerialInTransaction
|
||||
|
||||
@zope.interface.implementer(ZODB.interfaces.IStorageTransactionInformation)
|
||||
class TransactionRecord(TransactionMetaData):
|
||||
"""Abstract base class for iterator protocol"""
|
||||
|
||||
|
||||
def __init__(self, tid, status, user, description, extension):
|
||||
self.tid = tid
|
||||
self.status = status
|
||||
TransactionMetaData.__init__(self, user, description, extension)
|
||||
|
||||
@zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
|
||||
class DataRecord(object):
|
||||
"""Abstract base class for iterator protocol"""
|
||||
|
||||
|
||||
version = ''
|
||||
|
||||
def __init__(self, oid, tid, data, prev):
|
||||
self.oid = oid
|
||||
self.tid = tid
|
||||
self.data = data
|
||||
self.data_txn = prev
|
||||
@@ -0,0 +1,315 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
import logging
|
||||
|
||||
import six
|
||||
import zope.interface
|
||||
from ZODB.POSException import ConflictError
|
||||
from ZODB.loglevels import BLATHER
|
||||
from ZODB._compat import (
|
||||
BytesIO, PersistentUnpickler, PersistentPickler, _protocol)
|
||||
|
||||
# Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError,
|
||||
# and these are unrelated classes! So we shouldn't use pickle.PicklingError,
|
||||
# since on Python 2, ZODB._compat.pickle is cPickle.
|
||||
from pickle import PicklingError
|
||||
|
||||
|
||||
logger = logging.getLogger('ZODB.ConflictResolution')
|
||||
|
||||
class BadClassName(Exception):
|
||||
pass
|
||||
|
||||
class BadClass(object):
|
||||
|
||||
def __init__(self, *args):
|
||||
self.args = args
|
||||
|
||||
def __reduce__(self):
|
||||
raise BadClassName(*self.args)
|
||||
|
||||
_class_cache = {}
|
||||
_class_cache_get = _class_cache.get
|
||||
def find_global(*args):
|
||||
cls = _class_cache_get(args, 0)
|
||||
if cls == 0:
|
||||
# Not cached. Try to import
|
||||
try:
|
||||
module = __import__(args[0], {}, {}, ['cluck'])
|
||||
except ImportError:
|
||||
cls = 1
|
||||
else:
|
||||
cls = getattr(module, args[1], 1)
|
||||
_class_cache[args] = cls
|
||||
|
||||
if cls == 1:
|
||||
logger.log(BLATHER, "Unable to load class", exc_info=True)
|
||||
|
||||
if cls == 1:
|
||||
# Not importable
|
||||
if (isinstance(args, tuple) and len(args) == 2 and
|
||||
isinstance(args[0], six.string_types) and
|
||||
isinstance(args[1], six.string_types)
|
||||
):
|
||||
return BadClass(*args)
|
||||
else:
|
||||
raise BadClassName(*args)
|
||||
return cls
|
||||
|
||||
def state(self, oid, serial, prfactory, p=''):
|
||||
p = p or self.loadSerial(oid, serial)
|
||||
p = self._crs_untransform_record_data(p)
|
||||
file = BytesIO(p)
|
||||
unpickler = PersistentUnpickler(
|
||||
find_global, prfactory.persistent_load, file)
|
||||
unpickler.load() # skip the class tuple
|
||||
return unpickler.load()
|
||||
|
||||
class IPersistentReference(zope.interface.Interface):
|
||||
'''public contract for references to persistent objects from an object
|
||||
with conflicts.'''
|
||||
|
||||
oid = zope.interface.Attribute(
|
||||
'The oid of the persistent object that this reference represents')
|
||||
|
||||
database_name = zope.interface.Attribute(
|
||||
'''The name of the database of the reference, *if* different.
|
||||
|
||||
If not different, None.''')
|
||||
|
||||
klass = zope.interface.Attribute(
|
||||
'''class meta data. Presence is not reliable.''')
|
||||
|
||||
weak = zope.interface.Attribute(
|
||||
'''bool: whether this reference is weak''')
|
||||
|
||||
def __cmp__(other):
|
||||
'''if other is equivalent reference, return 0; else raise ValueError.
|
||||
|
||||
Equivalent in this case means that oid and database_name are the same.
|
||||
|
||||
If either is a weak reference, we only support `is` equivalence, and
|
||||
otherwise raise a ValueError even if the datbase_names and oids are
|
||||
the same, rather than guess at the correct semantics.
|
||||
|
||||
It is impossible to sort reliably, since the actual persistent
|
||||
class may have its own comparison, and we have no idea what it is.
|
||||
We assert that it is reasonably safe to assume that an object is
|
||||
equivalent to itself, but that's as much as we can say.
|
||||
|
||||
We don't compare on 'is other', despite the
|
||||
PersistentReferenceFactory.data cache, because it is possible to
|
||||
have two references to the same object that are spelled with different
|
||||
data (for instance, one with a class and one without).'''
|
||||
|
||||
@zope.interface.implementer(IPersistentReference)
|
||||
class PersistentReference(object):
|
||||
|
||||
|
||||
weak = False
|
||||
oid = database_name = klass = None
|
||||
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
# see serialize.py, ObjectReader._persistent_load
|
||||
if isinstance(data, tuple):
|
||||
self.oid, klass = data
|
||||
if isinstance(klass, BadClass):
|
||||
# We can't use the BadClass directly because, if
|
||||
# resolution succeeds, there's no good way to pickle
|
||||
# it. Fortunately, a class reference in a persistent
|
||||
# reference is allowed to be a module+name tuple.
|
||||
self.data = self.oid, klass.args
|
||||
elif isinstance(data, (bytes, str)):
|
||||
self.oid = data
|
||||
else: # a list
|
||||
reference_type = data[0]
|
||||
# 'm' = multi_persistent: (database_name, oid, klass)
|
||||
# 'n' = multi_oid: (database_name, oid)
|
||||
# 'w' = persistent weakref: (oid)
|
||||
# or persistent weakref: (oid, database_name)
|
||||
# else it is a weakref: reference_type
|
||||
if reference_type == 'm':
|
||||
self.database_name, self.oid, klass = data[1]
|
||||
if isinstance(klass, BadClass):
|
||||
# see above wrt BadClass
|
||||
data[1] = self.database_name, self.oid, klass.args
|
||||
elif reference_type == 'n':
|
||||
self.database_name, self.oid = data[1]
|
||||
elif reference_type == 'w':
|
||||
try:
|
||||
self.oid, = data[1]
|
||||
except ValueError:
|
||||
self.oid, self.database_name = data[1]
|
||||
self.weak = True
|
||||
else:
|
||||
assert len(data) == 1, 'unknown reference format'
|
||||
self.oid = data[0]
|
||||
self.weak = True
|
||||
if not isinstance(self.oid, (bytes, type(None))):
|
||||
assert isinstance(self.oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
self.oid = self.oid.encode('ascii')
|
||||
|
||||
def __cmp__(self, other):
|
||||
if self is other or (
|
||||
isinstance(other, PersistentReference) and
|
||||
self.oid == other.oid and
|
||||
self.database_name == other.database_name and
|
||||
not self.weak and
|
||||
not other.weak):
|
||||
return 0
|
||||
else:
|
||||
raise ValueError(
|
||||
"can't reliably compare against different "
|
||||
"PersistentReferences")
|
||||
|
||||
# Python 3 dropped __cmp__
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.__cmp__(other) == 0
|
||||
|
||||
def __ne__(self, other):
|
||||
return self.__cmp__(other) != 0
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.__cmp__(other) < 0
|
||||
|
||||
def __gt__(self, other):
|
||||
return self.__cmp__(other) > 0
|
||||
|
||||
def __le__(self, other):
|
||||
return self.__cmp__(other) <= 0
|
||||
|
||||
def __ge__(self, other):
|
||||
return self.__cmp__(other) >= 0
|
||||
|
||||
def __repr__(self):
|
||||
return "PR(%s %s)" % (id(self), self.data)
|
||||
|
||||
def __getstate__(self):
|
||||
raise PicklingError("Can't pickle PersistentReference")
|
||||
|
||||
@property
|
||||
def klass(self):
|
||||
# for tests
|
||||
data = self.data
|
||||
if isinstance(data, tuple):
|
||||
return data[1]
|
||||
elif isinstance(data, list) and data[0] == 'm':
|
||||
return data[1][2]
|
||||
|
||||
class PersistentReferenceFactory(object):
|
||||
|
||||
data = None
|
||||
|
||||
def persistent_load(self, ref):
|
||||
if self.data is None:
|
||||
self.data = {}
|
||||
key = tuple(ref) # lists are not hashable; formats are different enough
|
||||
# even after eliminating list/tuple distinction
|
||||
r = self.data.get(key, None)
|
||||
if r is None:
|
||||
r = PersistentReference(ref)
|
||||
self.data[key] = r
|
||||
|
||||
return r
|
||||
|
||||
def persistent_id(object):
|
||||
if getattr(object, '__class__', 0) is not PersistentReference:
|
||||
return None
|
||||
return object.data
|
||||
|
||||
_unresolvable = {}
|
||||
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
|
||||
committedData=b''):
|
||||
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
|
||||
klass = 'n/a'
|
||||
try:
|
||||
prfactory = PersistentReferenceFactory()
|
||||
newpickle = self._crs_untransform_record_data(newpickle)
|
||||
file = BytesIO(newpickle)
|
||||
unpickler = PersistentUnpickler(
|
||||
find_global, prfactory.persistent_load, file)
|
||||
meta = unpickler.load()
|
||||
if isinstance(meta, tuple):
|
||||
klass = meta[0]
|
||||
newargs = meta[1] or ()
|
||||
if isinstance(klass, tuple):
|
||||
klass = find_global(*klass)
|
||||
else:
|
||||
klass = meta
|
||||
newargs = ()
|
||||
|
||||
if klass in _unresolvable:
|
||||
raise ConflictError
|
||||
|
||||
inst = klass.__new__(klass, *newargs)
|
||||
|
||||
try:
|
||||
resolve = inst._p_resolveConflict
|
||||
except AttributeError:
|
||||
_unresolvable[klass] = 1
|
||||
raise ConflictError
|
||||
|
||||
|
||||
oldData = self.loadSerial(oid, oldSerial)
|
||||
if not committedData:
|
||||
committedData = self.loadSerial(oid, committedSerial)
|
||||
|
||||
newstate = unpickler.load()
|
||||
old = state(self, oid, oldSerial, prfactory, oldData)
|
||||
committed = state(self, oid, committedSerial, prfactory, committedData)
|
||||
|
||||
resolved = resolve(old, committed, newstate)
|
||||
|
||||
file = BytesIO()
|
||||
pickler = PersistentPickler(persistent_id, file, _protocol)
|
||||
pickler.dump(meta)
|
||||
pickler.dump(resolved)
|
||||
return self._crs_transform_record_data(file.getvalue())
|
||||
except (ConflictError, BadClassName) as e:
|
||||
logger.debug(
|
||||
"Conflict resolution on %s failed with %s: %s",
|
||||
klass, e.__class__.__name__, str(e))
|
||||
except:
|
||||
# If anything else went wrong, catch it here and avoid passing an
|
||||
# arbitrary exception back to the client. The error here will mask
|
||||
# the original ConflictError. A client can recover from a
|
||||
# ConflictError, but not necessarily from other errors. But log
|
||||
# the error so that any problems can be fixed.
|
||||
logger.exception(
|
||||
"Unexpected error while trying to resolve conflict on %s", klass)
|
||||
|
||||
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
|
||||
data=newpickle)
|
||||
|
||||
class ConflictResolvingStorage(object):
|
||||
"Mix-in class that provides conflict resolution handling for storages"
|
||||
|
||||
tryToResolveConflict = tryToResolveConflict
|
||||
|
||||
_crs_transform_record_data = _crs_untransform_record_data = (
|
||||
lambda self, o: o)
|
||||
|
||||
def registerDB(self, wrapper):
|
||||
self._crs_untransform_record_data = wrapper.untransform_record_data
|
||||
self._crs_transform_record_data = wrapper.transform_record_data
|
||||
try:
|
||||
m = super(ConflictResolvingStorage, self).registerDB
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
m(wrapper)
|
||||
@@ -0,0 +1,579 @@
|
||||
===================
|
||||
Conflict Resolution
|
||||
===================
|
||||
|
||||
Overview
|
||||
========
|
||||
|
||||
Conflict resolution is a way to resolve transaction conflicts that would
|
||||
otherwise abort a transaction. As such, it risks data integrity in order to
|
||||
try to avoid throwing away potentially computationally expensive transactions.
|
||||
|
||||
The risk of harming data integrity should not be underestimated. Writing
|
||||
conflict resolution code takes some responsibility for transactional
|
||||
integrity away from the ZODB, and puts it in the hands of the developer
|
||||
writing the conflict resolution code.
|
||||
|
||||
The current conflict resolution code is implemented with a storage mix-in
|
||||
found in ZODB/ConflictResolution.py. The idea's proposal, and an explanation
|
||||
of the interface, can be found here:
|
||||
http://www.zope.org/Members/jim/ZODB/ApplicationLevelConflictResolution
|
||||
|
||||
Here is the most pertinent section, somewhat modified for this document's
|
||||
use:
|
||||
|
||||
A new interface is proposed to allow object authors to provide a method
|
||||
for resolving conflicts. When a conflict is detected, then the database
|
||||
checks to see if the class of the object being saved defines the method,
|
||||
_p_resolveConflict. If the method is defined, then the method is called
|
||||
on the object. If the method succeeds, then the object change can be
|
||||
committed, otherwise a ConflictError is raised as usual.
|
||||
|
||||
def _p_resolveConflict(oldState, savedState, newState):
|
||||
Return the state of the object after resolving different changes.
|
||||
|
||||
Arguments:
|
||||
|
||||
oldState
|
||||
The state of the object that the changes made by the current
|
||||
transaction were based on.
|
||||
|
||||
The method is permitted to modify this value.
|
||||
|
||||
savedState
|
||||
The state of the object that is currently stored in the
|
||||
database. This state was written after oldState and reflects
|
||||
changes made by a transaction that committed before the
|
||||
current transaction.
|
||||
|
||||
The method is permitted to modify this value.
|
||||
|
||||
newState
|
||||
The state after changes made by the current transaction.
|
||||
|
||||
The method is not permitted to modify this value.
|
||||
|
||||
This method should compute a new state by merging changes
|
||||
reflected in savedState and newState, relative to oldState.
|
||||
|
||||
If the method cannot resolve the changes, then it should raise
|
||||
ZODB.POSException.ConflictError.
|
||||
|
||||
|
||||
Consider an extremely simple example, a counter::
|
||||
|
||||
from persistent import Persistent
|
||||
class PCounter(Persistent):
|
||||
'`value` is readonly; increment it with `inc`.'
|
||||
|
||||
# Fool BTree checks for sane comparison :/
|
||||
def __cmp__(self, other):
|
||||
return object.__cmp__(self, other)
|
||||
def __lt__(self, other):
|
||||
return object.__lt__(self, other)
|
||||
|
||||
_val = 0
|
||||
def inc(self):
|
||||
self._val += 1
|
||||
@property
|
||||
def value(self):
|
||||
return self._val
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
oldState['_val'] = (
|
||||
savedState.get('_val', 0) +
|
||||
newState.get('_val', 0) -
|
||||
oldState.get('_val', 0))
|
||||
return oldState
|
||||
|
||||
.. -> src
|
||||
|
||||
>>> import ConflictResolution_txt
|
||||
>>> exec(src, ConflictResolution_txt.__dict__)
|
||||
>>> PCounter = ConflictResolution_txt.PCounter
|
||||
>>> PCounter.__module__ = 'ConflictResolution_txt'
|
||||
|
||||
|
||||
By "state", the excerpt above means the value used by __getstate__ and
|
||||
__setstate__: a dictionary, in most cases. We'll look at more details below,
|
||||
but let's continue the example above with a simple successful resolution
|
||||
story.
|
||||
|
||||
First we create a storage and a database, and put a PCounter in the database.
|
||||
|
||||
>>> import ZODB
|
||||
>>> db = ZODB.DB('Data.fs')
|
||||
>>> import transaction
|
||||
>>> tm_A = transaction.TransactionManager()
|
||||
>>> conn_A = db.open(transaction_manager=tm_A)
|
||||
>>> p_A = conn_A.root()['p'] = PCounter()
|
||||
>>> p_A.value
|
||||
0
|
||||
>>> tm_A.commit()
|
||||
|
||||
Now get another copy of 'p' so we can make a conflict. Think of `conn_A`
|
||||
(connection A) as one thread, and `conn_B` (connection B) as a concurrent
|
||||
thread. `p_A` is a view on the object in the first connection, and `p_B`
|
||||
is a view on *the same persistent object* in the second connection.
|
||||
|
||||
>>> tm_B = transaction.TransactionManager()
|
||||
>>> conn_B = db.open(transaction_manager=tm_B)
|
||||
>>> p_B = conn_B.root()['p']
|
||||
>>> p_B.value
|
||||
0
|
||||
>>> p_A._p_oid == p_B._p_oid
|
||||
True
|
||||
|
||||
Now we can make a conflict, and see it resolved.
|
||||
|
||||
>>> p_A.inc()
|
||||
>>> p_A.value
|
||||
1
|
||||
>>> p_B.inc()
|
||||
>>> p_B.value
|
||||
1
|
||||
>>> tm_B.commit()
|
||||
>>> p_B.value
|
||||
1
|
||||
>>> tm_A.commit()
|
||||
>>> p_A.value
|
||||
2
|
||||
|
||||
We need to synchronize connection B, in any of a variety of ways, to see the
|
||||
change from connection A.
|
||||
|
||||
>>> p_B.value
|
||||
1
|
||||
>>> trans = tm_B.begin()
|
||||
>>> p_B.value
|
||||
2
|
||||
|
||||
A very similar class found in real world use is BTrees.Length.Length.
|
||||
|
||||
This conflict resolution approach is simple, yet powerful. However, it
|
||||
has a few caveats and rough edges in practice. The simplicity, then, is
|
||||
a bit of a disguise. Again, be warned, writing conflict resolution code
|
||||
means that you claim significant responsibilty for your data integrity.
|
||||
|
||||
Because of the rough edges, the current conflict resolution approach is slated
|
||||
for change (as of this writing, according to Jim Fulton, the ZODB
|
||||
primary author and maintainer). Others have talked about different approaches
|
||||
as well (see, for instance, http://www.python.org/~jeremy/weblog/031031c.html).
|
||||
But for now, the _p_resolveConflict method is what we have.
|
||||
|
||||
Caveats and Dangers
|
||||
===================
|
||||
|
||||
Here are caveats for working with this conflict resolution approach.
|
||||
Each sub-section has a "DANGERS" section that outlines what might happen
|
||||
if you ignore the warning. We work from the least danger to the most.
|
||||
|
||||
Conflict Resolution Is on the Server
|
||||
------------------------------------
|
||||
|
||||
If you are using ZEO or ZRS, be aware that the classes for which you have
|
||||
conflict resolution code *and* the classes of the non-persistent objects
|
||||
they reference must be available to import by the *server* (or ZRS
|
||||
primary).
|
||||
|
||||
DANGERS: You think you are going to get conflict resolution, but you won't.
|
||||
|
||||
Ignore `self`
|
||||
-------------
|
||||
|
||||
Even though the _p_resolveConflict method has a "self", ignore it.
|
||||
Don't change it. You make changes by returning the state. This is
|
||||
effectively a class method.
|
||||
|
||||
DANGERS: The changes you make to the instance will be discarded. The
|
||||
instance is not initialized, so other methods that depend on instance
|
||||
attributes will not work.
|
||||
|
||||
Here's an example of a broken _p_resolveConflict method::
|
||||
|
||||
class PCounter2(PCounter):
|
||||
def __init__(self):
|
||||
self.data = []
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
self.data.append('bad idea')
|
||||
return super(PCounter2, self)._p_resolveConflict(
|
||||
oldState, savedState, newState)
|
||||
|
||||
.. -> src
|
||||
|
||||
>>> exec(src, ConflictResolution_txt.__dict__)
|
||||
>>> PCounter2 = ConflictResolution_txt.PCounter2
|
||||
>>> PCounter2.__module__ = 'ConflictResolution_txt'
|
||||
|
||||
Now we'll prepare for the conflict again.
|
||||
|
||||
>>> p2_A = conn_A.root()['p2'] = PCounter2()
|
||||
>>> p2_A.value
|
||||
0
|
||||
>>> tm_A.commit()
|
||||
>>> trans = tm_B.begin() # sync
|
||||
>>> p2_B = conn_B.root()['p2']
|
||||
>>> p2_B.value
|
||||
0
|
||||
>>> p2_A._p_oid == p2_B._p_oid
|
||||
True
|
||||
|
||||
And now we will make a conflict.
|
||||
|
||||
>>> p2_A.inc()
|
||||
>>> p2_A.value
|
||||
1
|
||||
>>> p2_B.inc()
|
||||
>>> p2_B.value
|
||||
1
|
||||
>>> tm_B.commit()
|
||||
>>> p2_B.value
|
||||
1
|
||||
>>> tm_A.commit() # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ConflictError: database conflict error...
|
||||
|
||||
oops!
|
||||
|
||||
>>> tm_A.abort()
|
||||
>>> p2_A.value
|
||||
1
|
||||
>>> trans = tm_B.begin()
|
||||
>>> p2_B.value
|
||||
1
|
||||
|
||||
Watch Out for Persistent Objects in the State
|
||||
---------------------------------------------
|
||||
|
||||
If the object state has a reference to Persistent objects (instances
|
||||
of classes that inherit from persistent.Persistent) then these references
|
||||
*will not be loaded and are inaccessible*. Instead, persistent objects
|
||||
in the state dictionary are ZODB.ConflictResolution.PersistentReference
|
||||
instances. These objects have the following interface::
|
||||
|
||||
class IPersistentReference(zope.interface.Interface):
|
||||
'''public contract for references to persistent objects from an object
|
||||
with conflicts.'''
|
||||
|
||||
oid = zope.interface.Attribute(
|
||||
'The oid of the persistent object that this reference represents')
|
||||
|
||||
database_name = zope.interface.Attribute(
|
||||
'''The name of the database of the reference, *if* different.
|
||||
|
||||
If not different, None.''')
|
||||
|
||||
klass = zope.interface.Attribute(
|
||||
'''class meta data. Presence is not reliable.''')
|
||||
|
||||
weak = zope.interface.Attribute(
|
||||
'''bool: whether this reference is weak''')
|
||||
|
||||
def __cmp__(other):
|
||||
'''if other is equivalent reference, return 0; else raise ValueError.
|
||||
|
||||
Equivalent in this case means that oid and database_name are the same.
|
||||
|
||||
If either is a weak reference, we only support `is` equivalence, and
|
||||
otherwise raise a ValueError even if the datbase_names and oids are
|
||||
the same, rather than guess at the correct semantics.
|
||||
|
||||
It is impossible to sort reliably, since the actual persistent
|
||||
class may have its own comparison, and we have no idea what it is.
|
||||
We assert that it is reasonably safe to assume that an object is
|
||||
equivalent to itself, but that's as much as we can say.
|
||||
|
||||
We don't compare on 'is other', despite the
|
||||
PersistentReferenceFactory.data cache, because it is possible to
|
||||
have two references to the same object that are spelled with different
|
||||
data (for instance, one with a class and one without).'''
|
||||
|
||||
So let's look at one of these. Let's assume we have three, `old`,
|
||||
`saved`, and `new`, each representing a persistent reference to the same
|
||||
object within a _p_resolveConflict call from the oldState, savedState,
|
||||
and newState [#get_persistent_reference]_. They have an oid, `weak` is
|
||||
False, and `database_name` is None. `klass` happens to be set but this is
|
||||
not always the case.
|
||||
|
||||
>>> isinstance(new.oid, bytes)
|
||||
True
|
||||
>>> new.weak
|
||||
False
|
||||
>>> print(new.database_name)
|
||||
None
|
||||
>>> new.klass is PCounter
|
||||
True
|
||||
|
||||
There are a few subtleties to highlight here. First, notice that the
|
||||
database_name is only present if this is a cross-database reference
|
||||
(see cross-database-references.txt in this directory, and examples
|
||||
below). The database name and oid is sometimes a reasonable way to
|
||||
reliably sort Persistent objects (see zope.app.keyreference, for
|
||||
instance) but if your code compares one PersistentReference with a
|
||||
database_name and another without, you need to refuse to give an answer
|
||||
and raise an exception, because you can't know how the unknown
|
||||
database_name sorts.
|
||||
|
||||
We already saw a persistent reference with a database_name of None. Now
|
||||
let's suppose `new` is an example of a cross-database reference from a
|
||||
database named '2' [#cross-database]_.
|
||||
|
||||
>>> new.database_name
|
||||
'2'
|
||||
|
||||
As seen, the database_name is available for this cross-database reference,
|
||||
and not for others. References to persistent objects, as defined in
|
||||
seialize.py, have other variations, such as weak references, which are
|
||||
handled but not discussed here [#instantiation_test]_
|
||||
|
||||
Second, notice the __cmp__ behavior [#cmp_test]_. This is new behavior
|
||||
after ZODB 3.8 and addresses a serious problem for when persistent
|
||||
objects are compared in an _p_resolveConflict, such as that in the ZODB
|
||||
BTrees code. Prior to this change, it was not safe to use Persistent
|
||||
objects as keys in a BTree. You needed to define a __cmp__ for them to
|
||||
be sorted reliably out of the context of conflict resolution, but then
|
||||
during conflict resolution the sorting would be arbitrary, on the basis
|
||||
of the persistent reference's memory location. This could have lead to
|
||||
inconsistent state for BTrees (or BTree module buckets or tree sets or sets).
|
||||
|
||||
Here's an example of how the new behavior stops potentially incorrect
|
||||
resolution.
|
||||
|
||||
>>> import BTrees
|
||||
>>> treeset_A = conn_A.root()['treeset'] = BTrees.family32.OI.TreeSet()
|
||||
>>> tm_A.commit()
|
||||
>>> trans = tm_B.begin() # sync
|
||||
>>> treeset_B = conn_B.root()['treeset']
|
||||
>>> treeset_A.insert(PCounter())
|
||||
1
|
||||
>>> treeset_B.insert(PCounter())
|
||||
1
|
||||
>>> tm_B.commit()
|
||||
>>> tm_A.commit() # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ConflictError: database conflict error...
|
||||
>>> tm_A.abort()
|
||||
|
||||
Third, note that, even if the persistent object to which the reference refers
|
||||
changes in the same transaction, the reference is still the same.
|
||||
|
||||
DANGERS: subtle and potentially serious. Beyond the two subtleties above,
|
||||
which should now be addressed, there is a general problem for objects that
|
||||
are composites of smaller persistent objects--for instance, a BTree, in
|
||||
which the BTree and each bucket is a persistent object; or a
|
||||
zc.queue.CompositePersistentQueue, which is a persistent queue of
|
||||
persistent queues. Consider the following situation. It is actually solved,
|
||||
but it is a concrete example of what might go wrong.
|
||||
|
||||
A BTree (persistent object) has a two buckets (persistent objects). The
|
||||
second bucket has one persistent object in it. Concurrently, one thread
|
||||
deletes the one object in the second bucket, which causes the BTree to dump
|
||||
the bucket; and another thread puts an object in the second bucket. What
|
||||
happens during conflict resolution? Remember, each persistent object cannot
|
||||
see the other. From the perspective of the BTree object, it has no
|
||||
conflicts: one transaction modified it, causing it to lose a bucket; and the
|
||||
other transaction did not change it. From the perspective of the bucket,
|
||||
one transaction deleted an object and the other added it: it will resolve
|
||||
conflicts and say that the bucket has the new object and not the old one.
|
||||
However, it will be garbage collected, and effectively the addition of the
|
||||
new object will be lost.
|
||||
|
||||
As mentioned, this story is actually solved for BTrees. As
|
||||
BTrees/MergeTemplate.c explains, whenever savedState or newState for a bucket
|
||||
shows an empty bucket, the code refuses to resolve the conflict: this avoids
|
||||
the situation above.
|
||||
|
||||
>>> bucket_A = conn_A.root()['bucket'] = BTrees.family32.II.Bucket()
|
||||
>>> bucket_A[0] = 255
|
||||
>>> tm_A.commit()
|
||||
>>> trans = tm_B.begin() # sync
|
||||
>>> bucket_B = conn_B.root()['bucket']
|
||||
>>> bucket_B[1] = 254
|
||||
>>> del bucket_A[0]
|
||||
>>> tm_B.commit()
|
||||
>>> tm_A.commit() # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ConflictError: database conflict error...
|
||||
>>> tm_A.abort()
|
||||
|
||||
However, the story highlights the kinds of subtle problems that units
|
||||
made up of multiple composite Persistent objects need to contemplate.
|
||||
Any structure made up of objects that contain persistent objects with
|
||||
conflict resolution code, as a catalog index is made up of multiple
|
||||
BTree Buckets and Sets, each with conflict resolution, needs to think
|
||||
through these kinds of problems or be faced with potential data
|
||||
integrity issues.
|
||||
|
||||
.. cleanup
|
||||
|
||||
>>> db.close()
|
||||
>>> db1.close()
|
||||
>>> db2.close()
|
||||
|
||||
.. ......... ..
|
||||
.. FOOTNOTES ..
|
||||
.. ......... ..
|
||||
|
||||
.. [#get_persistent_reference] We'll catch persistent references with a class
|
||||
mutable.
|
||||
|
||||
::
|
||||
|
||||
class PCounter3(PCounter):
|
||||
data = []
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
PCounter3.data.append(
|
||||
(oldState.get('other'),
|
||||
savedState.get('other'),
|
||||
newState.get('other')))
|
||||
return super(PCounter3, self)._p_resolveConflict(
|
||||
oldState, savedState, newState)
|
||||
|
||||
.. -> src
|
||||
|
||||
>>> exec(src, ConflictResolution_txt.__dict__)
|
||||
>>> PCounter3 = ConflictResolution_txt.PCounter3
|
||||
>>> PCounter3.__module__ = 'ConflictResolution_txt'
|
||||
|
||||
>>> p3_A = conn_A.root()['p3'] = PCounter3()
|
||||
>>> p3_A.other = conn_A.root()['p']
|
||||
>>> tm_A.commit()
|
||||
>>> trans = tm_B.begin() # sync
|
||||
>>> p3_B = conn_B.root()['p3']
|
||||
>>> p3_A.inc()
|
||||
>>> p3_B.inc()
|
||||
>>> tm_B.commit()
|
||||
>>> tm_A.commit()
|
||||
>>> old, saved, new = PCounter3.data[-1]
|
||||
|
||||
.. [#cross-database] We need a whole different set of databases for this.
|
||||
See cross-database-references.txt in this directory for a discussion of
|
||||
what is going on here.
|
||||
|
||||
>>> databases = {}
|
||||
>>> db1 = ZODB.DB('1', databases=databases, database_name='1')
|
||||
>>> db2 = ZODB.DB('2', databases=databases, database_name='2')
|
||||
>>> tm_multi_A = transaction.TransactionManager()
|
||||
>>> conn_1A = db1.open(transaction_manager=tm_multi_A)
|
||||
>>> conn_2A = conn_1A.get_connection('2')
|
||||
>>> p4_1A = conn_1A.root()['p4'] = PCounter3()
|
||||
>>> p5_2A = conn_2A.root()['p5'] = PCounter3()
|
||||
>>> conn_2A.add(p5_2A)
|
||||
>>> p4_1A.other = p5_2A
|
||||
>>> tm_multi_A.commit()
|
||||
>>> tm_multi_B = transaction.TransactionManager()
|
||||
>>> conn_1B = db1.open(transaction_manager=tm_multi_B)
|
||||
>>> p4_1B = conn_1B.root()['p4']
|
||||
>>> p4_1A.inc()
|
||||
>>> p4_1B.inc()
|
||||
>>> tm_multi_B.commit()
|
||||
>>> tm_multi_A.commit()
|
||||
>>> old, saved, new = PCounter3.data[-1]
|
||||
|
||||
.. [#instantiation_test] We'll simply instantiate PersistentReferences
|
||||
with examples of types described in ZODB/serialize.py.
|
||||
|
||||
>>> from ZODB.ConflictResolution import PersistentReference
|
||||
|
||||
>>> ref1 = PersistentReference(b'my_oid')
|
||||
>>> ref1.oid
|
||||
'my_oid'
|
||||
>>> print(ref1.klass)
|
||||
None
|
||||
>>> print(ref1.database_name)
|
||||
None
|
||||
>>> ref1.weak
|
||||
False
|
||||
|
||||
>>> ref2 = PersistentReference((b'my_oid', 'my_class'))
|
||||
>>> ref2.oid
|
||||
'my_oid'
|
||||
>>> ref2.klass
|
||||
'my_class'
|
||||
>>> print(ref2.database_name)
|
||||
None
|
||||
>>> ref2.weak
|
||||
False
|
||||
|
||||
>>> ref3 = PersistentReference(['w', (b'my_oid',)])
|
||||
>>> ref3.oid
|
||||
'my_oid'
|
||||
>>> print(ref3.klass)
|
||||
None
|
||||
>>> print(ref3.database_name)
|
||||
None
|
||||
>>> ref3.weak
|
||||
True
|
||||
|
||||
>>> ref3a = PersistentReference(['w', (b'my_oid', 'other_db')])
|
||||
>>> ref3a.oid
|
||||
'my_oid'
|
||||
>>> print(ref3a.klass)
|
||||
None
|
||||
>>> ref3a.database_name
|
||||
'other_db'
|
||||
>>> ref3a.weak
|
||||
True
|
||||
|
||||
>>> ref4 = PersistentReference(['m', ('other_db', b'my_oid', 'my_class')])
|
||||
>>> ref4.oid
|
||||
'my_oid'
|
||||
>>> ref4.klass
|
||||
'my_class'
|
||||
>>> ref4.database_name
|
||||
'other_db'
|
||||
>>> ref4.weak
|
||||
False
|
||||
|
||||
>>> ref5 = PersistentReference(['n', ('other_db', b'my_oid')])
|
||||
>>> ref5.oid
|
||||
'my_oid'
|
||||
>>> print(ref5.klass)
|
||||
None
|
||||
>>> ref5.database_name
|
||||
'other_db'
|
||||
>>> ref5.weak
|
||||
False
|
||||
|
||||
>>> ref6 = PersistentReference([b'my_oid']) # legacy
|
||||
>>> ref6.oid
|
||||
'my_oid'
|
||||
>>> print(ref6.klass)
|
||||
None
|
||||
>>> print(ref6.database_name)
|
||||
None
|
||||
>>> ref6.weak
|
||||
True
|
||||
|
||||
.. [#cmp_test] All references are equal to themselves.
|
||||
|
||||
>>> ref1 == ref1 and ref2 == ref2 and ref4 == ref4 and ref5 == ref5
|
||||
True
|
||||
>>> ref3 == ref3 and ref3a == ref3a and ref6 == ref6 # weak references
|
||||
True
|
||||
|
||||
Non-weak references with the same oid and database_name are equal.
|
||||
|
||||
>>> ref1 == ref2 and ref4 == ref5
|
||||
True
|
||||
|
||||
Everything else raises a ValueError: weak references with the same oid and
|
||||
database, and references with a different database_name or oid.
|
||||
|
||||
>>> ref3 == ref6
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: can't reliably compare against different PersistentReferences
|
||||
|
||||
>>> ref1 == PersistentReference(('another_oid', 'my_class'))
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: can't reliably compare against different PersistentReferences
|
||||
|
||||
>>> ref4 == PersistentReference(
|
||||
... ['m', ('another_db', 'my_oid', 'my_class')])
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: can't reliably compare against different PersistentReferences
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,447 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) Zope Corporation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Demo ZODB storage
|
||||
|
||||
A demo storage supports demos by allowing a volatile changed database
|
||||
to be layered over a base database.
|
||||
|
||||
The base storage must not change.
|
||||
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import random
|
||||
import weakref
|
||||
import tempfile
|
||||
import ZODB.BaseStorage
|
||||
import ZODB.blob
|
||||
import ZODB.interfaces
|
||||
import ZODB.MappingStorage
|
||||
import ZODB.POSException
|
||||
import ZODB.utils
|
||||
import zope.interface
|
||||
|
||||
from .ConflictResolution import ConflictResolvingStorage
|
||||
from .utils import load_current, maxtid
|
||||
|
||||
@zope.interface.implementer(
|
||||
ZODB.interfaces.IStorage,
|
||||
ZODB.interfaces.IStorageIteration,
|
||||
)
|
||||
class DemoStorage(ConflictResolvingStorage):
|
||||
"""A storage that stores changes against a read-only base database
|
||||
|
||||
This storage was originally meant to support distribution of
|
||||
application demonstrations with populated read-only databases (on
|
||||
CDROM) and writable in-memory databases.
|
||||
|
||||
Demo storages are extemely convenient for testing where setup of a
|
||||
base database can be shared by many tests.
|
||||
|
||||
Demo storages are also handy for staging appplications where a
|
||||
read-only snapshot of a production database (often accomplished
|
||||
using a `beforestorage
|
||||
<https://pypi.python.org/pypi/zc.beforestorage>`_) is combined
|
||||
with a changes database implemented with a
|
||||
:class:`~ZODB.FileStorage.FileStorage.FileStorage`.
|
||||
"""
|
||||
|
||||
def __init__(self, name=None, base=None, changes=None,
|
||||
close_base_on_close=None, close_changes_on_close=None):
|
||||
"""Create a demo storage
|
||||
|
||||
:param str name: The storage name used by the
|
||||
:meth:`~ZODB.interfaces.IStorage.getName` and
|
||||
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
|
||||
:param object base: base storage
|
||||
:param object changes: changes storage
|
||||
:param bool close_base_on_close: A Flag indicating whether the base
|
||||
database should be closed when the demo storage is closed.
|
||||
:param bool close_changes_on_close: A Flag indicating whether the
|
||||
changes database should be closed when the demo storage is closed.
|
||||
|
||||
If a base database isn't provided, a
|
||||
:class:`~ZODB.MappingStorage.MappingStorage` will be
|
||||
constructed and used.
|
||||
|
||||
If ``close_base_on_close`` isn't specified, it will be ``True`` if
|
||||
a base database was provided and ``False`` otherwise.
|
||||
|
||||
If a changes database isn't provided, a
|
||||
:class:`~ZODB.MappingStorage.MappingStorage` will be
|
||||
constructed and used and blob support will be provided using a
|
||||
temporary blob directory.
|
||||
|
||||
If ``close_changes_on_close`` isn't specified, it will be ``True`` if
|
||||
a changes database was provided and ``False`` otherwise.
|
||||
"""
|
||||
|
||||
if close_base_on_close is None:
|
||||
if base is None:
|
||||
base = ZODB.MappingStorage.MappingStorage()
|
||||
close_base_on_close = False
|
||||
else:
|
||||
close_base_on_close = True
|
||||
elif base is None:
|
||||
base = ZODB.MappingStorage.MappingStorage()
|
||||
|
||||
self.base = base
|
||||
self.close_base_on_close = close_base_on_close
|
||||
|
||||
|
||||
if changes is None:
|
||||
self._temporary_changes = True
|
||||
changes = ZODB.MappingStorage.MappingStorage()
|
||||
zope.interface.alsoProvides(self, ZODB.interfaces.IBlobStorage)
|
||||
if close_changes_on_close is None:
|
||||
close_changes_on_close = False
|
||||
else:
|
||||
if ZODB.interfaces.IBlobStorage.providedBy(changes):
|
||||
zope.interface.alsoProvides(self, ZODB.interfaces.IBlobStorage)
|
||||
if close_changes_on_close is None:
|
||||
close_changes_on_close = True
|
||||
|
||||
self.changes = changes
|
||||
self.close_changes_on_close = close_changes_on_close
|
||||
|
||||
self._issued_oids = set()
|
||||
self._stored_oids = set()
|
||||
self._resolved = []
|
||||
|
||||
self._commit_lock = ZODB.utils.Lock()
|
||||
self._transaction = None
|
||||
|
||||
if name is None:
|
||||
name = 'DemoStorage(%r, %r)' % (base.getName(), changes.getName())
|
||||
self.__name__ = name
|
||||
|
||||
self._copy_methods_from_changes(changes)
|
||||
|
||||
self._next_oid = random.randint(1, 1<<62)
|
||||
|
||||
def _blobify(self):
|
||||
if (self._temporary_changes and
|
||||
isinstance(self.changes, ZODB.MappingStorage.MappingStorage)
|
||||
):
|
||||
blob_dir = tempfile.mkdtemp('.demoblobs')
|
||||
_temporary_blobdirs[
|
||||
weakref.ref(self, cleanup_temporary_blobdir)
|
||||
] = blob_dir
|
||||
self.changes = ZODB.blob.BlobStorage(blob_dir, self.changes)
|
||||
self._copy_methods_from_changes(self.changes)
|
||||
return True
|
||||
|
||||
def cleanup(self):
|
||||
self.base.cleanup()
|
||||
self.changes.cleanup()
|
||||
|
||||
__opened = True
|
||||
def opened(self):
|
||||
return self.__opened
|
||||
|
||||
def close(self):
|
||||
self.__opened = False
|
||||
if self.close_base_on_close:
|
||||
self.base.close()
|
||||
if self.close_changes_on_close:
|
||||
self.changes.close()
|
||||
|
||||
def _copy_methods_from_changes(self, changes):
|
||||
for meth in (
|
||||
'_lock',
|
||||
'getSize', 'isReadOnly',
|
||||
'sortKey', 'tpc_transaction',
|
||||
):
|
||||
setattr(self, meth, getattr(changes, meth))
|
||||
|
||||
supportsUndo = getattr(changes, 'supportsUndo', None)
|
||||
if supportsUndo is not None and supportsUndo():
|
||||
for meth in ('supportsUndo', 'undo', 'undoLog', 'undoInfo'):
|
||||
setattr(self, meth, getattr(changes, meth))
|
||||
zope.interface.alsoProvides(self, ZODB.interfaces.IStorageUndoable)
|
||||
|
||||
lastInvalidations = getattr(changes, 'lastInvalidations', None)
|
||||
if lastInvalidations is not None:
|
||||
self.lastInvalidations = lastInvalidations
|
||||
|
||||
def getName(self):
|
||||
return self.__name__
|
||||
__repr__ = getName
|
||||
|
||||
def getTid(self, oid):
|
||||
try:
|
||||
return self.changes.getTid(oid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
return self.base.getTid(oid)
|
||||
|
||||
def history(self, oid, size=1):
|
||||
try:
|
||||
r = self.changes.history(oid, size)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
r = []
|
||||
size -= len(r)
|
||||
if size:
|
||||
try:
|
||||
r += self.base.history(oid, size)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
if not r:
|
||||
raise
|
||||
return r
|
||||
|
||||
def iterator(self, start=None, end=None):
|
||||
for t in self.base.iterator(start, end):
|
||||
yield t
|
||||
for t in self.changes.iterator(start, end):
|
||||
yield t
|
||||
|
||||
def lastTransaction(self):
|
||||
t = self.changes.lastTransaction()
|
||||
if t == ZODB.utils.z64:
|
||||
t = self.base.lastTransaction()
|
||||
return t
|
||||
|
||||
def __len__(self):
|
||||
return len(self.changes)
|
||||
|
||||
# still want load for old clients (e.g. zeo servers)
|
||||
load = load_current
|
||||
|
||||
def loadBefore(self, oid, tid):
|
||||
try:
|
||||
result = self.changes.loadBefore(oid, tid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
# The oid isn't in the changes, so defer to base
|
||||
return self.base.loadBefore(oid, tid)
|
||||
|
||||
if result is None:
|
||||
# The oid *was* in the changes, but there aren't any
|
||||
# earlier records. Maybe there are in the base.
|
||||
try:
|
||||
result = self.base.loadBefore(oid, tid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
# The oid isn't in the base, so None will be the right result
|
||||
pass
|
||||
else:
|
||||
if result and not result[-1]:
|
||||
# The oid is current in the base. We need to find
|
||||
# the end tid in the base by fining the first tid
|
||||
# in the changes. Unfortunately, there isn't an
|
||||
# api for this, so we have to walk back using
|
||||
# loadBefore.
|
||||
|
||||
if tid == maxtid:
|
||||
# Special case: we were looking for the
|
||||
# current value. We won't find anything in
|
||||
# changes, so we're done.
|
||||
return result
|
||||
|
||||
end_tid = maxtid
|
||||
t = self.changes.loadBefore(oid, end_tid)
|
||||
while t:
|
||||
end_tid = t[1]
|
||||
t = self.changes.loadBefore(oid, end_tid)
|
||||
result = result[:2] + (
|
||||
end_tid if end_tid != maxtid else None,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def loadBlob(self, oid, serial):
|
||||
try:
|
||||
return self.changes.loadBlob(oid, serial)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
try:
|
||||
return self.base.loadBlob(oid, serial)
|
||||
except AttributeError:
|
||||
if not ZODB.interfaces.IBlobStorage.providedBy(self.base):
|
||||
raise ZODB.POSException.POSKeyError(oid, serial)
|
||||
raise
|
||||
except AttributeError:
|
||||
if self._blobify():
|
||||
return self.loadBlob(oid, serial)
|
||||
raise
|
||||
|
||||
def openCommittedBlobFile(self, oid, serial, blob=None):
|
||||
try:
|
||||
return self.changes.openCommittedBlobFile(oid, serial, blob)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
try:
|
||||
return self.base.openCommittedBlobFile(oid, serial, blob)
|
||||
except AttributeError:
|
||||
if not ZODB.interfaces.IBlobStorage.providedBy(self.base):
|
||||
raise ZODB.POSException.POSKeyError(oid, serial)
|
||||
raise
|
||||
except AttributeError:
|
||||
if self._blobify():
|
||||
return self.openCommittedBlobFile(oid, serial, blob)
|
||||
raise
|
||||
|
||||
def loadSerial(self, oid, serial):
|
||||
try:
|
||||
return self.changes.loadSerial(oid, serial)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
return self.base.loadSerial(oid, serial)
|
||||
|
||||
def new_oid(self):
|
||||
with self._lock:
|
||||
while 1:
|
||||
oid = ZODB.utils.p64(self._next_oid )
|
||||
if oid not in self._issued_oids:
|
||||
try:
|
||||
load_current(self.changes, oid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
try:
|
||||
load_current(self.base, oid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
self._next_oid += 1
|
||||
self._issued_oids.add(oid)
|
||||
return oid
|
||||
|
||||
self._next_oid = random.randint(1, 1<<62)
|
||||
|
||||
def pack(self, t, referencesf, gc=None):
|
||||
if gc is None:
|
||||
if self._temporary_changes:
|
||||
return self.changes.pack(t, referencesf)
|
||||
elif self._temporary_changes:
|
||||
return self.changes.pack(t, referencesf, gc=gc)
|
||||
elif gc:
|
||||
raise TypeError(
|
||||
"Garbage collection isn't supported"
|
||||
" when there is a base storage.")
|
||||
|
||||
try:
|
||||
self.changes.pack(t, referencesf, gc=False)
|
||||
except TypeError as v:
|
||||
if 'gc' in str(v):
|
||||
pass # The gc arg isn't supported. Don't pack
|
||||
raise
|
||||
|
||||
def pop(self):
|
||||
"""Close the changes database and return the base.
|
||||
"""
|
||||
self.changes.close()
|
||||
return self.base
|
||||
|
||||
def push(self, changes=None):
|
||||
"""Create a new demo storage using the storage as a base.
|
||||
|
||||
The given changes are used as the changes for the returned
|
||||
storage and ``False`` is passed as ``close_base_on_close``.
|
||||
"""
|
||||
return self.__class__(base=self, changes=changes,
|
||||
close_base_on_close=False)
|
||||
|
||||
def store(self, oid, serial, data, version, transaction):
|
||||
assert version=='', "versions aren't supported"
|
||||
if transaction is not self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(self, transaction)
|
||||
|
||||
# Since the OID is being used, we don't have to keep up with it any
|
||||
# more. Save it now so we can forget it later. :)
|
||||
self._stored_oids.add(oid)
|
||||
|
||||
# See if we already have changes for this oid
|
||||
try:
|
||||
old = load_current(self, oid)[1]
|
||||
except ZODB.POSException.POSKeyError:
|
||||
old = serial
|
||||
|
||||
if old != serial:
|
||||
rdata = self.tryToResolveConflict(oid, old, serial, data)
|
||||
self.changes.store(oid, old, rdata, '', transaction)
|
||||
self._resolved.append(oid)
|
||||
else:
|
||||
self.changes.store(oid, serial, data, '', transaction)
|
||||
|
||||
def storeBlob(self, oid, oldserial, data, blobfilename, version,
|
||||
transaction):
|
||||
assert version=='', "versions aren't supported"
|
||||
if transaction is not self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(self, transaction)
|
||||
|
||||
# Since the OID is being used, we don't have to keep up with it any
|
||||
# more. Save it now so we can forget it later. :)
|
||||
self._stored_oids.add(oid)
|
||||
|
||||
try:
|
||||
self.changes.storeBlob(
|
||||
oid, oldserial, data, blobfilename, '', transaction)
|
||||
except AttributeError:
|
||||
if not self._blobify():
|
||||
raise
|
||||
self.changes.storeBlob(
|
||||
oid, oldserial, data, blobfilename, '', transaction)
|
||||
|
||||
checkCurrentSerialInTransaction = (
|
||||
ZODB.BaseStorage.checkCurrentSerialInTransaction)
|
||||
|
||||
def temporaryDirectory(self):
|
||||
try:
|
||||
return self.changes.temporaryDirectory()
|
||||
except AttributeError:
|
||||
if self._blobify():
|
||||
return self.changes.temporaryDirectory()
|
||||
raise
|
||||
|
||||
def tpc_abort(self, transaction):
|
||||
with self._lock:
|
||||
if transaction is not self._transaction:
|
||||
return
|
||||
self._stored_oids = set()
|
||||
self._transaction = None
|
||||
self.changes.tpc_abort(transaction)
|
||||
self._commit_lock.release()
|
||||
|
||||
def tpc_begin(self, transaction, *a, **k):
|
||||
with self._lock:
|
||||
# The tid argument exists to support testing.
|
||||
if transaction is self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"Duplicate tpc_begin calls for same transaction")
|
||||
|
||||
self._commit_lock.acquire()
|
||||
|
||||
with self._lock:
|
||||
self.changes.tpc_begin(transaction, *a, **k)
|
||||
self._transaction = transaction
|
||||
self._stored_oids = set()
|
||||
del self._resolved[:]
|
||||
|
||||
def tpc_vote(self, *a, **k):
|
||||
if self.changes.tpc_vote(*a, **k):
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"Unexpected resolved conflicts")
|
||||
return self._resolved
|
||||
|
||||
def tpc_finish(self, transaction, func = lambda tid: None):
|
||||
with self._lock:
|
||||
if (transaction is not self._transaction):
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"tpc_finish called with wrong transaction")
|
||||
self._issued_oids.difference_update(self._stored_oids)
|
||||
self._stored_oids = set()
|
||||
self._transaction = None
|
||||
tid = self.changes.tpc_finish(transaction, func)
|
||||
self._commit_lock.release()
|
||||
return tid
|
||||
|
||||
_temporary_blobdirs = {}
|
||||
def cleanup_temporary_blobdir(
|
||||
ref,
|
||||
_temporary_blobdirs=_temporary_blobdirs, # Make sure it stays around
|
||||
):
|
||||
blob_dir = _temporary_blobdirs.pop(ref, None)
|
||||
if blob_dir and os.path.exists(blob_dir):
|
||||
ZODB.blob.remove_committed_dir(blob_dir)
|
||||
@@ -0,0 +1,468 @@
|
||||
==========================
|
||||
DemoStorage demo (doctest)
|
||||
==========================
|
||||
|
||||
DemoStorages provide a way to provide incremental updates to an
|
||||
existing, base, storage without updating the storage.
|
||||
|
||||
.. We need to mess with time to prevent spurious test failures on windows
|
||||
|
||||
>>> now = 1231019584.0
|
||||
>>> def faux_time_time():
|
||||
... global now
|
||||
... now += .1
|
||||
... return now
|
||||
>>> import time
|
||||
>>> real_time_time = time.time
|
||||
>>> if isinstance(time,type):
|
||||
... time.time = staticmethod(faux_time_time) # Jython
|
||||
... else:
|
||||
... time.time = faux_time_time
|
||||
|
||||
To see how this works, we'll start by creating a base storage and
|
||||
puting an object (in addition to the root object) in it:
|
||||
|
||||
>>> from ZODB.FileStorage import FileStorage
|
||||
>>> base = FileStorage('base.fs')
|
||||
>>> from ZODB.DB import DB
|
||||
>>> db = DB(base)
|
||||
>>> from persistent.mapping import PersistentMapping
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()['1'] = PersistentMapping({'a': 1, 'b':2})
|
||||
>>> import transaction
|
||||
>>> transaction.commit()
|
||||
>>> db.close()
|
||||
>>> import os
|
||||
>>> original_size = os.path.getsize('base.fs')
|
||||
|
||||
Now, lets reopen the base storage in read-only mode:
|
||||
|
||||
>>> base = FileStorage('base.fs', read_only=True)
|
||||
|
||||
And open a new storage to store changes:
|
||||
|
||||
>>> changes = FileStorage('changes.fs')
|
||||
|
||||
and combine the 2 in a demofilestorage:
|
||||
|
||||
>>> from ZODB.DemoStorage import DemoStorage
|
||||
>>> storage = DemoStorage(base=base, changes=changes)
|
||||
|
||||
The storage will assign OIDs in a pseudo-random fashion, but for test
|
||||
purposes we need to control where they start (since the random seeds
|
||||
can be different on different platforms):
|
||||
|
||||
>>> storage._next_oid = 3553260803050964942
|
||||
|
||||
|
||||
If there are no transactions, the storage reports the lastTransaction
|
||||
of the base database:
|
||||
|
||||
>>> storage.lastTransaction() == base.lastTransaction()
|
||||
True
|
||||
|
||||
Let's add some data:
|
||||
|
||||
>>> db = DB(storage)
|
||||
>>> conn = db.open()
|
||||
>>> items = sorted(conn.root()['1'].items())
|
||||
>>> items
|
||||
[('a', 1), ('b', 2)]
|
||||
|
||||
>>> conn.root()['2'] = PersistentMapping({'a': 3, 'b':4})
|
||||
>>> transaction.commit()
|
||||
|
||||
>>> conn.root()['2']['c'] = 5
|
||||
>>> transaction.commit()
|
||||
|
||||
Here we can see that we haven't modified the base storage:
|
||||
|
||||
>>> original_size == os.path.getsize('base.fs')
|
||||
True
|
||||
|
||||
But we have modified the changes database:
|
||||
|
||||
>>> len(changes)
|
||||
2
|
||||
|
||||
Our lastTransaction reflects the lastTransaction of the changes:
|
||||
|
||||
>>> storage.lastTransaction() > base.lastTransaction()
|
||||
True
|
||||
|
||||
>>> storage.lastTransaction() == changes.lastTransaction()
|
||||
True
|
||||
|
||||
Let's walk over some of the methods so we can see how we delegate to
|
||||
the new underlying storages:
|
||||
|
||||
>>> from ZODB.utils import p64, u64
|
||||
>>> storage.load(p64(0), '') == changes.load(p64(0), '')
|
||||
True
|
||||
>>> storage.load(p64(0), '') == base.load(p64(0), '')
|
||||
False
|
||||
>>> storage.load(p64(1), '') == base.load(p64(1), '')
|
||||
True
|
||||
|
||||
>>> serial = base.getTid(p64(0))
|
||||
>>> storage.loadSerial(p64(0), serial) == base.loadSerial(p64(0), serial)
|
||||
True
|
||||
|
||||
>>> serial = changes.getTid(p64(0))
|
||||
>>> storage.loadSerial(p64(0), serial) == changes.loadSerial(p64(0),
|
||||
... serial)
|
||||
True
|
||||
|
||||
The object id of the new object is quite random, and typically large:
|
||||
|
||||
>>> print(u64(conn.root()['2']._p_oid))
|
||||
3553260803050964942
|
||||
|
||||
Let's look at some other methods:
|
||||
|
||||
>>> storage.getName()
|
||||
"DemoStorage('base.fs', 'changes.fs')"
|
||||
|
||||
>>> storage.sortKey() == changes.sortKey()
|
||||
True
|
||||
|
||||
>>> storage.getSize() == changes.getSize()
|
||||
True
|
||||
|
||||
>>> len(storage) == len(changes)
|
||||
True
|
||||
|
||||
|
||||
Undo methods are simply copied from the changes storage:
|
||||
|
||||
>>> [getattr(storage, name) == getattr(changes, name)
|
||||
... for name in ('supportsUndo', 'undo', 'undoLog', 'undoInfo')
|
||||
... ]
|
||||
[True, True, True, True]
|
||||
|
||||
>>> db.close()
|
||||
|
||||
Closing demo storages
|
||||
=====================
|
||||
|
||||
Normally, when a demo storage is closed, it's base and changes
|
||||
storage are closed:
|
||||
|
||||
>>> from ZODB.MappingStorage import MappingStorage
|
||||
>>> demo = DemoStorage(base=MappingStorage(), changes=MappingStorage())
|
||||
>>> demo.close()
|
||||
>>> demo.base.opened(), demo.changes.opened()
|
||||
(False, False)
|
||||
|
||||
You can pass constructor arguments to control whether the base and
|
||||
changes storages should be closed when the demo storage is closed:
|
||||
|
||||
>>> demo = DemoStorage(
|
||||
... base=MappingStorage(), changes=MappingStorage(),
|
||||
... close_base_on_close=False, close_changes_on_close=False,
|
||||
... )
|
||||
>>> demo.close()
|
||||
>>> demo.base.opened(), demo.changes.opened()
|
||||
(True, True)
|
||||
|
||||
|
||||
Storage Stacking
|
||||
================
|
||||
|
||||
A common use case is to stack demo storages. DemoStorage provides
|
||||
some helper functions to help with this. The push method, just
|
||||
creates a new demo storage who's base is the original demo storage:
|
||||
|
||||
>>> demo = DemoStorage()
|
||||
>>> demo2 = demo.push()
|
||||
>>> demo2.base is demo
|
||||
True
|
||||
|
||||
We can also supply an explicit changes storage, if we wish:
|
||||
|
||||
>>> changes = MappingStorage()
|
||||
>>> demo3 = demo2.push(changes)
|
||||
>>> demo3.changes is changes, demo3.base is demo2
|
||||
(True, True)
|
||||
|
||||
The pop method closes the changes storage and returns the base
|
||||
*without* closing it:
|
||||
|
||||
>>> demo3.pop() is demo2
|
||||
True
|
||||
|
||||
>>> changes.opened()
|
||||
False
|
||||
|
||||
If storage returned by push is closed, the original storage isn't:
|
||||
|
||||
>>> demo3.push().close()
|
||||
>>> demo2.opened()
|
||||
True
|
||||
|
||||
Blob Support
|
||||
============
|
||||
|
||||
DemoStorage supports Blobs if the changes database supports blobs.
|
||||
|
||||
>>> import ZODB.blob
|
||||
>>> base = ZODB.blob.BlobStorage('base', FileStorage('base.fs'))
|
||||
>>> db = DB(base)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()['blob'] = ZODB.blob.Blob()
|
||||
>>> with conn.root()['blob'].open('w') as file:
|
||||
... _ = file.write(b'state 1')
|
||||
>>> transaction.commit()
|
||||
>>> db.close()
|
||||
|
||||
>>> base = ZODB.blob.BlobStorage('base',
|
||||
... FileStorage('base.fs', read_only=True))
|
||||
>>> changes = ZODB.blob.BlobStorage('changes',
|
||||
... FileStorage('changes.fs', create=True))
|
||||
>>> storage = DemoStorage(base=base, changes=changes)
|
||||
|
||||
>>> db = DB(storage)
|
||||
>>> conn = db.open()
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 1'
|
||||
>>> _ = transaction.begin()
|
||||
>>> with conn.root()['blob'].open('w') as file:
|
||||
... _ = file.write(b'state 2')
|
||||
>>> transaction.commit()
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 2'
|
||||
|
||||
>>> storage.temporaryDirectory() == changes.temporaryDirectory()
|
||||
True
|
||||
|
||||
>>> db.close()
|
||||
|
||||
It isn't necessary for the base database to support blobs.
|
||||
|
||||
>>> base = FileStorage('base.fs', read_only=True)
|
||||
>>> changes = ZODB.blob.BlobStorage('changes', FileStorage('changes.fs'))
|
||||
>>> storage = DemoStorage(base=base, changes=changes)
|
||||
>>> db = DB(storage)
|
||||
>>> conn = db.open()
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 2'
|
||||
|
||||
>>> _ = transaction.begin()
|
||||
>>> conn.root()['blob2'] = ZODB.blob.Blob()
|
||||
>>> with conn.root()['blob2'].open('w') as file:
|
||||
... _ = file.write(b'state 1')
|
||||
>>> with conn.root()['blob2'].open() as fp: fp.read()
|
||||
'state 1'
|
||||
|
||||
>>> db.close()
|
||||
|
||||
If the changes database is created implicitly, it will get a blob
|
||||
storage wrapped around it when necessary:
|
||||
|
||||
>>> base = ZODB.blob.BlobStorage('base',
|
||||
... FileStorage('base.fs', read_only=True))
|
||||
>>> storage = DemoStorage(base=base)
|
||||
|
||||
>>> type(storage.changes).__name__
|
||||
'MappingStorage'
|
||||
|
||||
>>> db = DB(storage)
|
||||
>>> conn = db.open()
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 1'
|
||||
|
||||
>>> type(storage.changes).__name__
|
||||
'BlobStorage'
|
||||
|
||||
>>> _ = transaction.begin()
|
||||
>>> with conn.root()['blob'].open('w') as file:
|
||||
... _ = file.write(b'state 2')
|
||||
>>> transaction.commit()
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 2'
|
||||
|
||||
>>> storage.temporaryDirectory() == storage.changes.temporaryDirectory()
|
||||
True
|
||||
|
||||
>>> db.close()
|
||||
|
||||
This works even if we first write a blob rather than read a blob:
|
||||
|
||||
>>> base = ZODB.blob.BlobStorage('base',
|
||||
... FileStorage('base.fs', read_only=True))
|
||||
>>> storage = DemoStorage(base=base)
|
||||
|
||||
>>> type(storage.changes).__name__
|
||||
'MappingStorage'
|
||||
|
||||
>>> db = DB(storage)
|
||||
>>> conn = db.open()
|
||||
|
||||
>>> _ = transaction.begin()
|
||||
>>> conn.root()['blob'] = ZODB.blob.Blob()
|
||||
>>> with conn.root()['blob'].open('w') as file:
|
||||
... _ = file.write(b'state 2')
|
||||
>>> transaction.commit()
|
||||
|
||||
>>> type(storage.changes).__name__
|
||||
'BlobStorage'
|
||||
|
||||
>>> with conn.root()['blob'].open() as fp: fp.read()
|
||||
'state 2'
|
||||
|
||||
>>> storage.temporaryDirectory() == storage.changes.temporaryDirectory()
|
||||
True
|
||||
|
||||
>>> db.close()
|
||||
|
||||
|
||||
.. Check that the temporary directory is gone
|
||||
|
||||
For now, it won't go until the storage does.
|
||||
|
||||
>>> transaction.abort()
|
||||
>>> blobdir = storage.temporaryDirectory()
|
||||
>>> del storage, _
|
||||
|
||||
>>> import gc
|
||||
>>> _ = gc.collect()
|
||||
|
||||
>>> import os
|
||||
>>> os.path.exists(blobdir)
|
||||
False
|
||||
|
||||
ZConfig support
|
||||
===============
|
||||
|
||||
You can configure demo storages using ZConfig, using name, changes,
|
||||
and base options:
|
||||
|
||||
>>> import ZODB.config
|
||||
>>> storage = ZODB.config.storageFromString("""
|
||||
... <demostorage>
|
||||
... </demostorage>
|
||||
... """)
|
||||
>>> storage.getName()
|
||||
"DemoStorage('MappingStorage', 'MappingStorage')"
|
||||
|
||||
>>> storage = ZODB.config.storageFromString("""
|
||||
... <demostorage>
|
||||
... <filestorage base>
|
||||
... path base.fs
|
||||
... </filestorage>
|
||||
...
|
||||
... <filestorage changes>
|
||||
... path changes.fs
|
||||
... </filestorage>
|
||||
... </demostorage>
|
||||
... """)
|
||||
>>> storage.getName()
|
||||
"DemoStorage('base.fs', 'changes.fs')"
|
||||
|
||||
>>> storage.close()
|
||||
|
||||
>>> storage = ZODB.config.storageFromString("""
|
||||
... <demostorage>
|
||||
... name bob
|
||||
... <filestorage>
|
||||
... path base.fs
|
||||
... </filestorage>
|
||||
...
|
||||
... <filestorage changes>
|
||||
... path changes.fs
|
||||
... </filestorage>
|
||||
... </demostorage>
|
||||
... """)
|
||||
>>> storage.getName()
|
||||
'bob'
|
||||
>>> storage.base.getName()
|
||||
'base.fs'
|
||||
|
||||
>>> storage.close()
|
||||
|
||||
Generating OIDs
|
||||
===============
|
||||
|
||||
When asked for a new OID DemoStorage chooses a value and then
|
||||
verifies that neither the base or changes storages already contain
|
||||
that OID. It chooses values sequentially from random starting
|
||||
points, picking new starting points whenever a chosen value us already
|
||||
in the changes or base.
|
||||
|
||||
Under rare circumstances an OID can be chosen that has already been
|
||||
handed out, but which hasn't yet been comitted. Lets verify that if
|
||||
the same OID is chosen twice during a transaction that everything will
|
||||
still work.
|
||||
|
||||
To test this, we need to hack random.randint a bit.
|
||||
|
||||
>>> import random
|
||||
>>> randint = random.randint
|
||||
|
||||
>>> rv = 42
|
||||
>>> def faux_randint(min, max):
|
||||
... print('called randint')
|
||||
... global rv
|
||||
... rv += 1000
|
||||
... return rv
|
||||
|
||||
>>> random.randint = faux_randint
|
||||
|
||||
Now, we create a demostorage.
|
||||
|
||||
>>> storage = DemoStorage()
|
||||
called randint
|
||||
|
||||
If we ask for an oid, we'll get 1042.
|
||||
|
||||
>>> print(u64(storage.new_oid()))
|
||||
1042
|
||||
|
||||
oids are allocated seuentially:
|
||||
|
||||
>>> print(u64(storage.new_oid()))
|
||||
1043
|
||||
|
||||
Now, we'll save 1044 in changes so that it has to pick a new one randomly.
|
||||
|
||||
>>> t = transaction.get()
|
||||
>>> ZODB.tests.util.store(storage.changes, 1044)
|
||||
|
||||
>>> print(u64(storage.new_oid()))
|
||||
called randint
|
||||
2042
|
||||
|
||||
Now, we hack rv to 1042 is given out again and we'll save 2043 in base
|
||||
to force another attempt:
|
||||
|
||||
>>> rv -= 1000
|
||||
>>> ZODB.tests.util.store(storage.changes, 2043)
|
||||
>>> oid = storage.new_oid()
|
||||
called randint
|
||||
called randint
|
||||
>>> print(u64(oid))
|
||||
3042
|
||||
|
||||
DemoStorage keeps up with the issued OIDs to know when not to reissue them...
|
||||
|
||||
>>> oid in storage._issued_oids
|
||||
True
|
||||
|
||||
...but once data is stored with a given OID...
|
||||
|
||||
>>> ZODB.tests.util.store(storage, oid)
|
||||
|
||||
...there's no need to remember it any longer:
|
||||
|
||||
>>> oid in storage._issued_oids
|
||||
False
|
||||
|
||||
>>> storage.close()
|
||||
|
||||
.. restore randint
|
||||
|
||||
>>> random.randint = randint
|
||||
|
||||
.. restore time
|
||||
|
||||
>>> time.time = real_time_time
|
||||
@@ -0,0 +1,208 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Support for database export and import."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
import six
|
||||
|
||||
from ZODB.blob import Blob
|
||||
from ZODB.interfaces import IBlobStorage
|
||||
from ZODB.POSException import ExportError
|
||||
from ZODB.serialize import referencesf
|
||||
from ZODB.utils import p64, u64, cp, mktemp
|
||||
from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol
|
||||
|
||||
|
||||
logger = logging.getLogger('ZODB.ExportImport')
|
||||
|
||||
class ExportImport(object):
|
||||
|
||||
def exportFile(self, oid, f=None, bufsize=64 * 1024):
|
||||
if f is None:
|
||||
f = TemporaryFile(prefix="EXP")
|
||||
elif isinstance(f, six.string_types):
|
||||
f = open(f,'w+b')
|
||||
f.write(b'ZEXP')
|
||||
oids = [oid]
|
||||
done_oids = {}
|
||||
done = done_oids.__contains__
|
||||
load = self._storage.load
|
||||
supports_blobs = IBlobStorage.providedBy(self._storage)
|
||||
while oids:
|
||||
oid = oids.pop(0)
|
||||
if oid in done_oids:
|
||||
continue
|
||||
done_oids[oid] = True
|
||||
try:
|
||||
p, serial = load(oid)
|
||||
except:
|
||||
logger.debug("broken reference for oid %s", repr(oid),
|
||||
exc_info=True)
|
||||
else:
|
||||
referencesf(p, oids)
|
||||
f.writelines([oid, p64(len(p)), p])
|
||||
|
||||
if supports_blobs:
|
||||
if not isinstance(self._reader.getGhost(p), Blob):
|
||||
continue # not a blob
|
||||
|
||||
blobfilename = self._storage.loadBlob(oid, serial)
|
||||
f.write(blob_begin_marker)
|
||||
f.write(p64(os.stat(blobfilename).st_size))
|
||||
blobdata = open(blobfilename, "rb")
|
||||
cp(blobdata, f, bufsize=bufsize)
|
||||
blobdata.close()
|
||||
|
||||
f.write(export_end_marker)
|
||||
return f
|
||||
|
||||
def importFile(self, f, clue='', customImporters=None):
|
||||
# This is tricky, because we need to work in a transaction!
|
||||
|
||||
if isinstance(f, six.string_types):
|
||||
with open(f, 'rb') as fp:
|
||||
return self.importFile(fp, clue=clue,
|
||||
customImporters=customImporters)
|
||||
|
||||
magic = f.read(4)
|
||||
if magic != b'ZEXP':
|
||||
if customImporters and magic in customImporters:
|
||||
f.seek(0)
|
||||
return customImporters[magic](self, f, clue)
|
||||
raise ExportError("Invalid export header")
|
||||
|
||||
t = self.transaction_manager.get()
|
||||
if clue:
|
||||
t.note(clue)
|
||||
|
||||
return_oid_list = []
|
||||
self._import = f, return_oid_list
|
||||
self._register()
|
||||
t.savepoint(optimistic=True)
|
||||
# Return the root imported object.
|
||||
if return_oid_list:
|
||||
return self.get(return_oid_list[0])
|
||||
else:
|
||||
return None
|
||||
|
||||
def _importDuringCommit(self, transaction, f, return_oid_list):
|
||||
"""Import data during two-phase commit.
|
||||
|
||||
Invoked by the transaction manager mid commit.
|
||||
Appends one item, the OID of the first object created,
|
||||
to return_oid_list.
|
||||
"""
|
||||
oids = {}
|
||||
|
||||
# IMPORTANT: This code should be consistent with the code in
|
||||
# serialize.py. It is currently out of date and doesn't handle
|
||||
# weak references.
|
||||
|
||||
def persistent_load(ooid):
|
||||
"""Remap a persistent id to a new ID and create a ghost for it."""
|
||||
|
||||
klass = None
|
||||
if isinstance(ooid, tuple):
|
||||
ooid, klass = ooid
|
||||
|
||||
if not isinstance(ooid, bytes):
|
||||
assert isinstance(ooid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
ooid = ooid.encode('ascii')
|
||||
|
||||
if ooid in oids:
|
||||
oid = oids[ooid]
|
||||
else:
|
||||
if klass is None:
|
||||
oid = self._storage.new_oid()
|
||||
else:
|
||||
oid = self._storage.new_oid(), klass
|
||||
oids[ooid] = oid
|
||||
|
||||
return Ghost(oid)
|
||||
|
||||
while 1:
|
||||
header = f.read(16)
|
||||
if header == export_end_marker:
|
||||
break
|
||||
if len(header) != 16:
|
||||
raise ExportError("Truncated export file")
|
||||
|
||||
# Extract header information
|
||||
ooid = header[:8]
|
||||
length = u64(header[8:16])
|
||||
data = f.read(length)
|
||||
|
||||
if len(data) != length:
|
||||
raise ExportError("Truncated export file")
|
||||
|
||||
if oids:
|
||||
oid = oids[ooid]
|
||||
if isinstance(oid, tuple):
|
||||
oid = oid[0]
|
||||
else:
|
||||
oids[ooid] = oid = self._storage.new_oid()
|
||||
return_oid_list.append(oid)
|
||||
|
||||
if (b'blob' in data and
|
||||
isinstance(self._reader.getGhost(data), Blob)
|
||||
):
|
||||
# Blob support
|
||||
|
||||
# Make sure we have a (redundant, overly) blob marker.
|
||||
if f.read(len(blob_begin_marker)) != blob_begin_marker:
|
||||
raise ValueError("No data for blob object")
|
||||
|
||||
# Copy the blob data to a temporary file
|
||||
# and remember the name
|
||||
blob_len = u64(f.read(8))
|
||||
blob_filename = mktemp(self._storage.temporaryDirectory())
|
||||
blob_file = open(blob_filename, "wb")
|
||||
cp(f, blob_file, blob_len)
|
||||
blob_file.close()
|
||||
else:
|
||||
blob_filename = None
|
||||
|
||||
pfile = BytesIO(data)
|
||||
unpickler = Unpickler(pfile)
|
||||
unpickler.persistent_load = persistent_load
|
||||
|
||||
newp = BytesIO()
|
||||
pickler = PersistentPickler(persistent_id, newp, _protocol)
|
||||
|
||||
pickler.dump(unpickler.load())
|
||||
pickler.dump(unpickler.load())
|
||||
data = newp.getvalue()
|
||||
|
||||
if blob_filename is not None:
|
||||
self._storage.storeBlob(oid, None, data, blob_filename,
|
||||
'', transaction)
|
||||
else:
|
||||
self._storage.store(oid, None, data, '', transaction)
|
||||
|
||||
|
||||
export_end_marker = b'\377'*16
|
||||
blob_begin_marker = b'\000BLOBSTART'
|
||||
|
||||
class Ghost(object):
|
||||
__slots__ = ("oid",)
|
||||
def __init__(self, oid):
|
||||
self.oid = oid
|
||||
|
||||
def persistent_id(obj):
|
||||
if isinstance(obj, Ghost):
|
||||
return obj.oid
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
# this is a package
|
||||
|
||||
from ZODB.FileStorage.FileStorage import FileStorage, TransactionRecord
|
||||
from ZODB.FileStorage.FileStorage import FileIterator, Record, packed_version
|
||||
|
||||
|
||||
# BBB Alias for compatibility
|
||||
RecordIterator = TransactionRecord
|
||||
@@ -0,0 +1,289 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
#
|
||||
# File-based ZODB storage
|
||||
#
|
||||
# Files are arranged as follows.
|
||||
#
|
||||
# - The first 4 bytes are a file identifier.
|
||||
#
|
||||
# - The rest of the file consists of a sequence of transaction
|
||||
# "records".
|
||||
#
|
||||
# A transaction record consists of:
|
||||
#
|
||||
# - 8-byte transaction id, which is also a time stamp.
|
||||
#
|
||||
# - 8-byte transaction record length - 8.
|
||||
#
|
||||
# - 1-byte status code
|
||||
# ' ' (a blank) completed transaction that hasn't been packed
|
||||
# 'p' completed transaction that has been packed
|
||||
# 'c' checkpoint -- a transaction in progress, at the end of the file;
|
||||
# it's been thru vote() but not finish(); if finish() completes
|
||||
# normally, it will be overwritten with a blank; if finish() dies
|
||||
# (e.g., out of disk space), cleanup code will try to truncate
|
||||
# the file to chop off this incomplete transaction
|
||||
# 'u' uncertain; no longer used; was previously used to record something
|
||||
# about non-transactional undo
|
||||
#
|
||||
# - 2-byte length of user name
|
||||
#
|
||||
# - 2-byte length of description
|
||||
#
|
||||
# - 2-byte length of extension attributes
|
||||
#
|
||||
# - user name
|
||||
#
|
||||
# - description
|
||||
#
|
||||
# - extension attributes
|
||||
#
|
||||
# * A sequence of data records
|
||||
#
|
||||
# - 8-byte redundant transaction length -8
|
||||
#
|
||||
# A data record consists of
|
||||
#
|
||||
# - 8-byte oid.
|
||||
#
|
||||
# - 8-byte tid, which matches the transaction id in the transaction record.
|
||||
#
|
||||
# - 8-byte previous-record file-position.
|
||||
#
|
||||
# - 8-byte beginning of transaction record file position.
|
||||
#
|
||||
# - 2-bytes with zero values. (Was version length.)
|
||||
#
|
||||
# - 8-byte data length
|
||||
#
|
||||
# ? data
|
||||
# (data length > 0)
|
||||
#
|
||||
# ? 8-byte position of data record containing data
|
||||
# (data length == 0)
|
||||
#
|
||||
# Note that the lengths and positions are all big-endian.
|
||||
# Also, the object ids time stamps are big-endian, so comparisons
|
||||
# are meaningful.
|
||||
#
|
||||
# Backpointers
|
||||
#
|
||||
# When we undo a record, we don't copy (or delete)
|
||||
# data. Instead, we write records with back pointers.
|
||||
|
||||
import logging
|
||||
import struct
|
||||
|
||||
from ZODB.POSException import POSKeyError
|
||||
from ZODB.utils import u64, oid_repr, as_bytes
|
||||
from ZODB._compat import PY3
|
||||
|
||||
class CorruptedError(Exception):
|
||||
pass
|
||||
|
||||
class CorruptedDataError(CorruptedError):
|
||||
|
||||
def __init__(self, oid=None, buf=None, pos=None):
|
||||
self.oid = oid
|
||||
self.buf = buf
|
||||
self.pos = pos
|
||||
|
||||
def __str__(self):
|
||||
if self.oid:
|
||||
msg = "Error reading oid %s. Found %r" % (oid_repr(self.oid),
|
||||
self.buf)
|
||||
else:
|
||||
msg = "Error reading unknown oid. Found %r" % self.buf
|
||||
if self.pos:
|
||||
msg += " at %d" % self.pos
|
||||
return msg
|
||||
|
||||
# the struct formats for the headers
|
||||
TRANS_HDR = ">8sQcHHH"
|
||||
DATA_HDR = ">8s8sQQHQ"
|
||||
# constants to support various header sizes
|
||||
TRANS_HDR_LEN = 23
|
||||
DATA_HDR_LEN = 42
|
||||
assert struct.calcsize(TRANS_HDR) == TRANS_HDR_LEN
|
||||
assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
|
||||
|
||||
logger = logging.getLogger('ZODB.FileStorage.format')
|
||||
|
||||
class FileStorageFormatter(object):
|
||||
"""Mixin class that can read and write the low-level format."""
|
||||
|
||||
# subclasses must provide _file
|
||||
|
||||
_metadata_size = 4
|
||||
_format_version = "21"
|
||||
|
||||
def _read_num(self, pos):
|
||||
"""Read an 8-byte number."""
|
||||
self._file.seek(pos)
|
||||
return u64(self._file.read(8))
|
||||
|
||||
def _read_data_header(self, pos, oid=None, _file=None):
|
||||
"""Return a DataHeader object for data record at pos.
|
||||
|
||||
If ois is not None, raise CorruptedDataError if oid passed
|
||||
does not match oid in file.
|
||||
"""
|
||||
if _file is None:
|
||||
_file = self._file
|
||||
|
||||
_file.seek(pos)
|
||||
s = _file.read(DATA_HDR_LEN)
|
||||
if len(s) != DATA_HDR_LEN:
|
||||
raise CorruptedDataError(oid, s, pos)
|
||||
h = DataHeaderFromString(s)
|
||||
if oid is not None and oid != h.oid:
|
||||
raise CorruptedDataError(oid, s, pos)
|
||||
if not h.plen:
|
||||
h.back = u64(_file.read(8))
|
||||
return h
|
||||
|
||||
def _read_txn_header(self, pos, tid=None):
|
||||
self._file.seek(pos)
|
||||
s = self._file.read(TRANS_HDR_LEN)
|
||||
if len(s) != TRANS_HDR_LEN:
|
||||
raise CorruptedDataError(tid, s, pos)
|
||||
h = TxnHeaderFromString(s)
|
||||
if tid is not None and tid != h.tid:
|
||||
raise CorruptedDataError(tid, s, pos)
|
||||
h.user = self._file.read(h.ulen)
|
||||
h.descr = self._file.read(h.dlen)
|
||||
h.ext = self._file.read(h.elen)
|
||||
return h
|
||||
|
||||
def _loadBack_impl(self, oid, back, fail=True, _file=None):
|
||||
# shared implementation used by various _loadBack methods
|
||||
#
|
||||
# If the backpointer ultimately resolves to 0:
|
||||
# If fail is True, raise KeyError for zero backpointer.
|
||||
# If fail is False, return the empty data from the record
|
||||
# with no backpointer.
|
||||
if _file is None:
|
||||
_file = self._file
|
||||
while 1:
|
||||
if not back:
|
||||
# If backpointer is 0, object does not currently exist.
|
||||
raise POSKeyError(oid)
|
||||
h = self._read_data_header(back, _file=_file)
|
||||
if h.plen:
|
||||
return _file.read(h.plen), h.tid, back, h.tloc
|
||||
if h.back == 0 and not fail:
|
||||
return None, h.tid, back, h.tloc
|
||||
back = h.back
|
||||
|
||||
def _loadBackTxn(self, oid, back, fail=True):
|
||||
"""Return data and txn id for backpointer."""
|
||||
return self._loadBack_impl(oid, back, fail)[:2]
|
||||
|
||||
def _loadBackPOS(self, oid, back):
|
||||
return self._loadBack_impl(oid, back)[2]
|
||||
|
||||
def getTxnFromData(self, oid, back):
|
||||
"""Return transaction id for data at back."""
|
||||
h = self._read_data_header(back, oid)
|
||||
return h.tid
|
||||
|
||||
def fail(self, pos, msg, *args):
|
||||
s = ("%s:%s:" + msg) % ((self._name, pos) + args)
|
||||
logger.error(s)
|
||||
raise CorruptedError(s)
|
||||
|
||||
def checkTxn(self, th, pos):
|
||||
if th.tid <= self.ltid:
|
||||
self.fail(pos, "time-stamp reduction: %s <= %s",
|
||||
oid_repr(th.tid), oid_repr(self.ltid))
|
||||
self.ltid = th.tid
|
||||
if th.status == "c":
|
||||
self.fail(pos, "transaction with checkpoint flag set")
|
||||
if not th.status in " pu": # recognize " ", "p", and "u" as valid
|
||||
self.fail(pos, "invalid transaction status: %r", th.status)
|
||||
if th.tlen < th.headerlen():
|
||||
self.fail(pos, "invalid transaction header: "
|
||||
"txnlen (%d) < headerlen(%d)", th.tlen, th.headerlen())
|
||||
|
||||
def checkData(self, th, tpos, dh, pos):
|
||||
if dh.tloc != tpos:
|
||||
self.fail(pos, "data record does not point to transaction header"
|
||||
": %d != %d", dh.tloc, tpos)
|
||||
if pos + dh.recordlen() > tpos + th.tlen:
|
||||
self.fail(pos, "data record size exceeds transaction size: "
|
||||
"%d > %d", pos + dh.recordlen(), tpos + th.tlen)
|
||||
if dh.prev >= pos:
|
||||
self.fail(pos, "invalid previous pointer: %d", dh.prev)
|
||||
if dh.back:
|
||||
if dh.back >= pos:
|
||||
self.fail(pos, "invalid back pointer: %d", dh.prev)
|
||||
if dh.plen:
|
||||
self.fail(pos, "data record has back pointer and data")
|
||||
|
||||
def DataHeaderFromString(s):
|
||||
return DataHeader(*struct.unpack(DATA_HDR, s))
|
||||
|
||||
class DataHeader(object):
|
||||
"""Header for a data record."""
|
||||
|
||||
__slots__ = ("oid", "tid", "prev", "tloc", "plen", "back")
|
||||
|
||||
def __init__(self, oid, tid, prev, tloc, vlen, plen):
|
||||
if vlen:
|
||||
raise ValueError(
|
||||
"Non-zero version length. Versions aren't supported.")
|
||||
|
||||
self.oid = oid
|
||||
self.tid = tid
|
||||
self.prev = prev
|
||||
self.tloc = tloc
|
||||
self.plen = plen
|
||||
self.back = 0 # default
|
||||
|
||||
def asString(self):
|
||||
return struct.pack(DATA_HDR, self.oid, self.tid, self.prev,
|
||||
self.tloc, 0, self.plen)
|
||||
|
||||
def recordlen(self):
|
||||
return DATA_HDR_LEN + (self.plen or 8)
|
||||
|
||||
def TxnHeaderFromString(s):
|
||||
res = TxnHeader(*struct.unpack(TRANS_HDR, s))
|
||||
if PY3:
|
||||
res.status = res.status.decode('ascii')
|
||||
return res
|
||||
|
||||
class TxnHeader(object):
|
||||
"""Header for a transaction record."""
|
||||
|
||||
__slots__ = ("tid", "tlen", "status", "user", "descr", "ext",
|
||||
"ulen", "dlen", "elen")
|
||||
|
||||
def __init__(self, tid, tlen, status, ulen, dlen, elen):
|
||||
self.tid = tid
|
||||
self.tlen = tlen
|
||||
self.status = status
|
||||
self.ulen = ulen
|
||||
self.dlen = dlen
|
||||
self.elen = elen
|
||||
assert elen >= 0
|
||||
|
||||
def asString(self):
|
||||
s = struct.pack(TRANS_HDR, self.tid, self.tlen, as_bytes(self.status),
|
||||
self.ulen, self.dlen, self.elen)
|
||||
return b"".join(map(as_bytes, [s, self.user, self.descr, self.ext]))
|
||||
|
||||
def headerlen(self):
|
||||
return TRANS_HDR_LEN + self.ulen + self.dlen + self.elen
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import print_function
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
import struct
|
||||
|
||||
from ZODB.FileStorage import FileIterator
|
||||
from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
|
||||
from ZODB.FileStorage.format import DATA_HDR, DATA_HDR_LEN
|
||||
from ZODB.TimeStamp import TimeStamp
|
||||
from ZODB.utils import u64, get_pickle_metadata
|
||||
|
||||
def fsdump(path, file=None, with_offset=1):
|
||||
iter = FileIterator(path)
|
||||
for i, trans in enumerate(iter):
|
||||
if with_offset:
|
||||
print(("Trans #%05d tid=%016x time=%s offset=%d" %
|
||||
(i, u64(trans.tid), TimeStamp(trans.tid), trans._pos)), file=file)
|
||||
else:
|
||||
print(("Trans #%05d tid=%016x time=%s" %
|
||||
(i, u64(trans.tid), TimeStamp(trans.tid))), file=file)
|
||||
print((" status=%r user=%r description=%r" %
|
||||
(trans.status, trans.user, trans.description)), file=file)
|
||||
|
||||
for j, rec in enumerate(trans):
|
||||
if rec.data is None:
|
||||
fullclass = "undo or abort of object creation"
|
||||
size = ""
|
||||
else:
|
||||
modname, classname = get_pickle_metadata(rec.data)
|
||||
size = " size=%d" % len(rec.data)
|
||||
fullclass = "%s.%s" % (modname, classname)
|
||||
|
||||
if rec.data_txn:
|
||||
# It would be nice to print the transaction number
|
||||
# (i) but it would be expensive to keep track of.
|
||||
bp = " bp=%016x" % u64(rec.data_txn)
|
||||
else:
|
||||
bp = ""
|
||||
|
||||
print((" data #%05d oid=%016x%s class=%s%s" %
|
||||
(j, u64(rec.oid), size, fullclass, bp)), file=file)
|
||||
iter.close()
|
||||
|
||||
def fmt(p64):
|
||||
# Return a nicely formatted string for a packaged 64-bit value
|
||||
return "%016x" % u64(p64)
|
||||
|
||||
class Dumper(object):
|
||||
"""A very verbose dumper for debuggin FileStorage problems."""
|
||||
|
||||
# TODO: Should revise this class to use FileStorageFormatter.
|
||||
|
||||
def __init__(self, path, dest=None):
|
||||
self.file = open(path, "rb")
|
||||
self.dest = dest
|
||||
|
||||
def dump(self):
|
||||
fid = self.file.read(4)
|
||||
print("*" * 60, file=self.dest)
|
||||
print("file identifier: %r" % fid, file=self.dest)
|
||||
while self.dump_txn():
|
||||
pass
|
||||
|
||||
def dump_txn(self):
|
||||
pos = self.file.tell()
|
||||
h = self.file.read(TRANS_HDR_LEN)
|
||||
if not h:
|
||||
return False
|
||||
tid, tlen, status, ul, dl, el = struct.unpack(TRANS_HDR, h)
|
||||
end = pos + tlen
|
||||
print("=" * 60, file=self.dest)
|
||||
print("offset: %d" % pos, file=self.dest)
|
||||
print("end pos: %d" % end, file=self.dest)
|
||||
print("transaction id: %s" % fmt(tid), file=self.dest)
|
||||
print("trec len: %d" % tlen, file=self.dest)
|
||||
print("status: %r" % status, file=self.dest)
|
||||
user = descr = extra = ""
|
||||
if ul:
|
||||
user = self.file.read(ul)
|
||||
if dl:
|
||||
descr = self.file.read(dl)
|
||||
if el:
|
||||
extra = self.file.read(el)
|
||||
print("user: %r" % user, file=self.dest)
|
||||
print("description: %r" % descr, file=self.dest)
|
||||
print("len(extra): %d" % el, file=self.dest)
|
||||
while self.file.tell() < end:
|
||||
self.dump_data(pos)
|
||||
stlen = self.file.read(8)
|
||||
print("redundant trec len: %d" % u64(stlen), file=self.dest)
|
||||
return 1
|
||||
|
||||
def dump_data(self, tloc):
|
||||
pos = self.file.tell()
|
||||
h = self.file.read(DATA_HDR_LEN)
|
||||
assert len(h) == DATA_HDR_LEN
|
||||
oid, revid, prev, tloc, vlen, dlen = struct.unpack(DATA_HDR, h)
|
||||
print("-" * 60, file=self.dest)
|
||||
print("offset: %d" % pos, file=self.dest)
|
||||
print("oid: %s" % fmt(oid), file=self.dest)
|
||||
print("revid: %s" % fmt(revid), file=self.dest)
|
||||
print("previous record offset: %d" % prev, file=self.dest)
|
||||
print("transaction offset: %d" % tloc, file=self.dest)
|
||||
assert not vlen
|
||||
print("len(data): %d" % dlen, file=self.dest)
|
||||
self.file.read(dlen)
|
||||
if not dlen:
|
||||
sbp = self.file.read(8)
|
||||
print("backpointer: %d" % u64(sbp), file=self.dest)
|
||||
|
||||
def main():
|
||||
import sys
|
||||
fsdump(sys.argv[1])
|
||||
@@ -0,0 +1,200 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
from __future__ import print_function
|
||||
import ZODB.FileStorage
|
||||
from ZODB.utils import get_pickle_metadata, p64, oid_repr, tid_repr
|
||||
from ZODB.serialize import get_refs
|
||||
from ZODB.TimeStamp import TimeStamp
|
||||
|
||||
# Extract module.class string from pickle.
|
||||
def get_class(pickle):
|
||||
return "%s.%s" % get_pickle_metadata(pickle)
|
||||
|
||||
# Shorten a string for display.
|
||||
def shorten(s, size=50):
|
||||
if len(s) <= size:
|
||||
return s
|
||||
# Stick ... in the middle.
|
||||
navail = size - 5
|
||||
nleading = navail // 2
|
||||
ntrailing = size - nleading
|
||||
return s[:nleading] + " ... " + s[-ntrailing:]
|
||||
|
||||
class Tracer(object):
|
||||
"""Trace all occurrences of a set of oids in a FileStorage.
|
||||
|
||||
Create passing a path to an existing FileStorage.
|
||||
Call register_oids(oid, ...) one or more times to specify which oids to
|
||||
investigate.
|
||||
Call run() to do the analysis. This isn't swift -- it has to read
|
||||
every byte in the database, in order to find all references.
|
||||
Call report() to display the results.
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
import os
|
||||
if not os.path.isfile(path):
|
||||
raise ValueError("must specify an existing FileStorage")
|
||||
self.path = path
|
||||
# Map an interesting tid to (status, user, description, pos).
|
||||
self.tid2info = {}
|
||||
# List of messages. Each is a tuple of the form
|
||||
# (oid, tid, string)
|
||||
# The order in the tuple is important, because it defines the
|
||||
# sort order for grouping.
|
||||
self.msgs = []
|
||||
# The set of interesting oids, specified by register_oid() calls.
|
||||
# Maps oid to # of revisions.
|
||||
self.oids = {}
|
||||
# Maps interesting oid to its module.class name. If a creation
|
||||
# record for an interesting oid is never seen, it won't appear
|
||||
# in this mapping.
|
||||
self.oid2name = {}
|
||||
|
||||
def register_oids(self, *oids):
|
||||
"""
|
||||
Declare that oids (0 or more) are "interesting".
|
||||
|
||||
An oid can be given as a native 8-byte string, or as an
|
||||
integer.
|
||||
|
||||
Info will be gathered about all appearances of this oid in the
|
||||
entire database, including references.
|
||||
"""
|
||||
for oid in oids:
|
||||
if isinstance(oid, bytes):
|
||||
assert len(oid) == 8
|
||||
else:
|
||||
oid = p64(oid)
|
||||
self.oids[oid] = 0 # 0 revisions seen so far
|
||||
|
||||
def _msg(self, oid, tid, *args):
|
||||
self.msgs.append( (oid, tid, ' '.join(map(str, args))) )
|
||||
self._produced_msg = True
|
||||
|
||||
def report(self):
|
||||
"""Show all msgs, grouped by oid and sub-grouped by tid."""
|
||||
|
||||
msgs = self.msgs
|
||||
oids = self.oids
|
||||
oid2name = self.oid2name
|
||||
# First determine which oids weren't seen at all, and synthesize msgs
|
||||
# for them.
|
||||
NOT_SEEN = "this oid was not defined (no data record for it found)"
|
||||
for oid in oids:
|
||||
if oid not in oid2name:
|
||||
msgs.append( (oid, None, NOT_SEEN) )
|
||||
|
||||
msgs.sort() # oids are primary key, tids secondary
|
||||
current_oid = current_tid = None
|
||||
for oid, tid, msg in msgs:
|
||||
if oid != current_oid:
|
||||
nrev = oids[oid]
|
||||
revision = "revision" + (nrev != 1 and 's' or '')
|
||||
name = oid2name.get(oid, "<unknown>")
|
||||
print("oid", oid_repr(oid), name, nrev, revision)
|
||||
current_oid = oid
|
||||
current_tid = None
|
||||
if msg is NOT_SEEN:
|
||||
assert tid is None
|
||||
print(" ", msg)
|
||||
continue
|
||||
if tid != current_tid:
|
||||
current_tid = tid
|
||||
status, user, description, pos = self.tid2info[tid]
|
||||
print(" tid %s offset=%d %s" % (tid_repr(tid),
|
||||
pos,
|
||||
TimeStamp(tid)))
|
||||
print(" tid user=%r" % shorten(user))
|
||||
print(" tid description=%r" % shorten(description))
|
||||
print(" ", msg)
|
||||
|
||||
# Do the analysis.
|
||||
def run(self):
|
||||
"""Find all occurrences of the registered oids in the database."""
|
||||
|
||||
# Maps oid of a reference to its module.class name.
|
||||
self._ref2name = {}
|
||||
for txn in ZODB.FileStorage.FileIterator(self.path):
|
||||
self._check_trec(txn)
|
||||
|
||||
# Process next transaction record.
|
||||
def _check_trec(self, txn):
|
||||
# txn has members tid, status, user, description,
|
||||
# _extension, _pos, _tend, _file, _tpos
|
||||
self._produced_msg = False
|
||||
# Map and list for save data records for current transaction.
|
||||
self._records_map = {}
|
||||
self._records = []
|
||||
for drec in txn:
|
||||
self._save_references(drec)
|
||||
for drec in self._records:
|
||||
self._check_drec(drec)
|
||||
if self._produced_msg:
|
||||
# Copy txn info for later output.
|
||||
self.tid2info[txn.tid] = (txn.status, txn.user, txn.description,
|
||||
txn._tpos)
|
||||
|
||||
def _save_references(self, drec):
|
||||
# drec has members oid, tid, data, data_txn
|
||||
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
|
||||
if pick:
|
||||
if oid in self.oids:
|
||||
klass = get_class(pick)
|
||||
self._msg(oid, tid, "new revision", klass, "at", pos)
|
||||
self.oids[oid] += 1
|
||||
self.oid2name[oid] = self._ref2name[oid] = klass
|
||||
self._records_map[oid] = drec
|
||||
self._records.append(drec)
|
||||
elif oid in self.oids:
|
||||
self._msg(oid, tid, "creation undo at", pos)
|
||||
|
||||
# Process next data record. If a message is produced, self._produced_msg
|
||||
# will be set True.
|
||||
def _check_drec(self, drec):
|
||||
# drec has members oid, tid, data, data_txn
|
||||
tid, oid, pick, pos = drec.tid, drec.oid, drec.data, drec.pos
|
||||
ref2name = self._ref2name
|
||||
ref2name_get = ref2name.get
|
||||
records_map_get = self._records_map.get
|
||||
if pick:
|
||||
oid_in_oids = oid in self.oids
|
||||
for ref, klass in get_refs(pick):
|
||||
if ref in self.oids:
|
||||
oidclass = ref2name_get(oid, None)
|
||||
if oidclass is None:
|
||||
ref2name[oid] = oidclass = get_class(pick)
|
||||
self._msg(ref, tid, "referenced by", oid_repr(oid),
|
||||
oidclass, "at", pos)
|
||||
|
||||
if oid_in_oids:
|
||||
if klass is None:
|
||||
klass = ref2name_get(ref, None)
|
||||
if klass is None:
|
||||
r = records_map_get(ref, None)
|
||||
# For save memory we only save references
|
||||
# seen in one transaction with interesting
|
||||
# objects changes. So in some circumstances
|
||||
# we may still got "<unknown>" class name.
|
||||
if r is None:
|
||||
klass = "<unknown>"
|
||||
else:
|
||||
ref2name[ref] = klass = get_class(r.data)
|
||||
elif isinstance(klass, tuple):
|
||||
ref2name[ref] = klass = "%s.%s" % klass
|
||||
else:
|
||||
klass = "%s.%s" % (klass.__module__, klass.__name__)
|
||||
|
||||
self._msg(oid, tid, "references", oid_repr(ref), klass,
|
||||
"at", pos)
|
||||
@@ -0,0 +1,668 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""FileStorage helper to perform pack.
|
||||
|
||||
A storage contains an ordered set of object revisions. When a storage
|
||||
is packed, object revisions that are not reachable as of the pack time
|
||||
are deleted. The notion of reachability is complicated by
|
||||
backpointers -- object revisions that point to earlier revisions of
|
||||
the same object.
|
||||
|
||||
An object revisions is reachable at a certain time if it is reachable
|
||||
from the revision of the root at that time or if it is reachable from
|
||||
a backpointer after that time.
|
||||
"""
|
||||
|
||||
from ZODB.FileStorage.format import DataHeader, TRANS_HDR_LEN
|
||||
from ZODB.FileStorage.format import FileStorageFormatter, CorruptedDataError
|
||||
from ZODB.utils import p64, u64, z64
|
||||
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import ZODB.fsIndex
|
||||
import ZODB.POSException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PackError(ZODB.POSException.POSError):
|
||||
pass
|
||||
|
||||
class PackCopier(FileStorageFormatter):
|
||||
|
||||
def __init__(self, f, index, tindex):
|
||||
self._file = f
|
||||
self._index = index
|
||||
self._tindex = tindex
|
||||
self._pos = None
|
||||
|
||||
def _txn_find(self, tid, stop_at_pack):
|
||||
# _pos always points just past the last transaction
|
||||
pos = self._pos
|
||||
while pos > 4:
|
||||
self._file.seek(pos - 8)
|
||||
pos = pos - u64(self._file.read(8)) - 8
|
||||
self._file.seek(pos)
|
||||
h = self._file.read(TRANS_HDR_LEN) # XXX bytes
|
||||
_tid = h[:8]
|
||||
if _tid == tid:
|
||||
return pos
|
||||
if stop_at_pack:
|
||||
if h[16] == 'p':
|
||||
break
|
||||
raise PackError("Invalid backpointer transaction id")
|
||||
|
||||
def _data_find(self, tpos, oid, data):
|
||||
# Return backpointer for oid. Must call with the lock held.
|
||||
# This is a file offset to oid's data record if found, else 0.
|
||||
# The data records in the transaction at tpos are searched for oid.
|
||||
# If a data record for oid isn't found, returns 0.
|
||||
# Else if oid's data record contains a backpointer, that
|
||||
# backpointer is returned.
|
||||
# Else oid's data record contains the data, and the file offset of
|
||||
# oid's data record is returned. This data record should contain
|
||||
# a pickle identical to the 'data' argument.
|
||||
|
||||
# Unclear: If the length of the stored data doesn't match len(data),
|
||||
# an exception is raised. If the lengths match but the data isn't
|
||||
# the same, 0 is returned. Why the discrepancy?
|
||||
h = self._read_txn_header(tpos)
|
||||
tend = tpos + h.tlen
|
||||
pos = self._file.tell()
|
||||
while pos < tend:
|
||||
h = self._read_data_header(pos)
|
||||
if h.oid == oid:
|
||||
# Make sure this looks like the right data record
|
||||
if h.plen == 0:
|
||||
# This is also a backpointer. Gotta trust it.
|
||||
return pos
|
||||
if h.plen != len(data):
|
||||
# The expected data doesn't match what's in the
|
||||
# backpointer. Something is wrong.
|
||||
logger.error("Mismatch between data and backpointer at %d",
|
||||
pos)
|
||||
return 0
|
||||
_data = self._file.read(h.plen)
|
||||
if data != _data:
|
||||
return 0
|
||||
return pos
|
||||
pos += h.recordlen()
|
||||
return 0
|
||||
|
||||
def copy(self, oid, serial, data, prev_txn, txnpos, datapos):
|
||||
prev_pos = self._resolve_backpointer(prev_txn, oid, data)
|
||||
old = self._index.get(oid, 0)
|
||||
# Calculate the pos the record will have in the storage.
|
||||
here = datapos
|
||||
# And update the temp file index
|
||||
self._tindex[oid] = here
|
||||
if prev_pos:
|
||||
# If there is a valid prev_pos, don't write data.
|
||||
data = None
|
||||
if data is None:
|
||||
dlen = 0
|
||||
else:
|
||||
dlen = len(data)
|
||||
# Write the recovery data record
|
||||
h = DataHeader(oid, serial, old, txnpos, 0, dlen)
|
||||
|
||||
self._file.write(h.asString())
|
||||
# Write the data or a backpointer
|
||||
if data is None:
|
||||
if prev_pos:
|
||||
self._file.write(p64(prev_pos))
|
||||
else:
|
||||
# Write a zero backpointer, which indicates an
|
||||
# un-creation transaction.
|
||||
self._file.write(z64)
|
||||
else:
|
||||
self._file.write(data)
|
||||
|
||||
def setTxnPos(self, pos):
|
||||
self._pos = pos
|
||||
|
||||
def _resolve_backpointer(self, prev_txn, oid, data):
|
||||
pos = self._file.tell()
|
||||
try:
|
||||
prev_pos = 0
|
||||
if prev_txn is not None:
|
||||
prev_txn_pos = self._txn_find(prev_txn, 0)
|
||||
if prev_txn_pos:
|
||||
prev_pos = self._data_find(prev_txn_pos, oid, data)
|
||||
return prev_pos
|
||||
finally:
|
||||
self._file.seek(pos)
|
||||
|
||||
class GC(FileStorageFormatter):
|
||||
|
||||
def __init__(self, file, eof, packtime, gc, referencesf):
|
||||
self._file = file
|
||||
self._name = file.name
|
||||
self.eof = eof
|
||||
self.packtime = packtime
|
||||
self.gc = gc
|
||||
# packpos: position of first txn header after pack time
|
||||
self.packpos = None
|
||||
|
||||
# {oid -> current data record position}:
|
||||
self.oid2curpos = ZODB.fsIndex.fsIndex()
|
||||
|
||||
# The set of reachable revisions of each object.
|
||||
#
|
||||
# This set as managed using two data structures. The first is
|
||||
# an fsIndex mapping oids to one data record pos. Since only
|
||||
# a few objects will have more than one revision, we use this
|
||||
# efficient data structure to handle the common case. The
|
||||
# second is a dictionary mapping objects to lists of
|
||||
# positions; it is used to handle the same number of objects
|
||||
# for which we must keep multiple revisions.
|
||||
self.reachable = ZODB.fsIndex.fsIndex()
|
||||
self.reach_ex = {}
|
||||
|
||||
# keep ltid for consistency checks during initial scan
|
||||
self.ltid = z64
|
||||
|
||||
self.referencesf = referencesf
|
||||
|
||||
def isReachable(self, oid, pos):
|
||||
"""Return 1 if revision of `oid` at `pos` is reachable."""
|
||||
|
||||
rpos = self.reachable.get(oid)
|
||||
if rpos is None:
|
||||
return 0
|
||||
if rpos == pos:
|
||||
return 1
|
||||
return pos in self.reach_ex.get(oid, [])
|
||||
|
||||
def findReachable(self):
|
||||
self.buildPackIndex()
|
||||
if self.gc:
|
||||
self.findReachableAtPacktime([z64])
|
||||
self.findReachableFromFuture()
|
||||
# These mappings are no longer needed and may consume a lot of
|
||||
# space.
|
||||
del self.oid2curpos
|
||||
else:
|
||||
self.reachable = self.oid2curpos
|
||||
|
||||
def buildPackIndex(self):
|
||||
pos = 4
|
||||
# We make the initial assumption that the database has been
|
||||
# packed before and set unpacked to True only after seeing the
|
||||
# first record with a status == " ". If we get to the packtime
|
||||
# and unpacked is still False, we need to watch for a redundant
|
||||
# pack.
|
||||
unpacked = False
|
||||
while pos < self.eof:
|
||||
th = self._read_txn_header(pos)
|
||||
if th.tid > self.packtime:
|
||||
break
|
||||
self.checkTxn(th, pos)
|
||||
if th.status != "p":
|
||||
unpacked = True
|
||||
|
||||
tpos = pos
|
||||
end = pos + th.tlen
|
||||
pos += th.headerlen()
|
||||
|
||||
while pos < end:
|
||||
dh = self._read_data_header(pos)
|
||||
self.checkData(th, tpos, dh, pos)
|
||||
if dh.plen or dh.back:
|
||||
self.oid2curpos[dh.oid] = pos
|
||||
else:
|
||||
if dh.oid in self.oid2curpos:
|
||||
del self.oid2curpos[dh.oid]
|
||||
pos += dh.recordlen()
|
||||
|
||||
tlen = self._read_num(pos)
|
||||
if tlen != th.tlen:
|
||||
self.fail(pos, "redundant transaction length does not "
|
||||
"match initial transaction length: %d != %d",
|
||||
tlen, th.tlen)
|
||||
pos += 8
|
||||
|
||||
self.packpos = pos
|
||||
|
||||
if unpacked:
|
||||
return
|
||||
# check for a redundant pack. If the first record following
|
||||
# the newly computed packpos has status 'p', then it was
|
||||
# packed earlier and the current pack is redudant.
|
||||
try:
|
||||
th = self._read_txn_header(pos)
|
||||
except CorruptedDataError as err:
|
||||
if err.buf != b"":
|
||||
raise
|
||||
if th.status == 'p':
|
||||
# Delayed import to cope with circular imports.
|
||||
# TODO: put exceptions in a separate module.
|
||||
from ZODB.FileStorage.FileStorage import RedundantPackWarning
|
||||
raise RedundantPackWarning(
|
||||
"The database has already been packed to a later time"
|
||||
" or no changes have been made since the last pack")
|
||||
|
||||
def findReachableAtPacktime(self, roots):
|
||||
"""Mark all objects reachable from the oids in roots as reachable."""
|
||||
reachable = self.reachable
|
||||
oid2curpos = self.oid2curpos
|
||||
|
||||
todo = list(roots)
|
||||
while todo:
|
||||
oid = todo.pop()
|
||||
if oid in reachable:
|
||||
continue
|
||||
|
||||
try:
|
||||
pos = oid2curpos[oid]
|
||||
except KeyError:
|
||||
if oid == z64 and len(oid2curpos) == 0:
|
||||
# special case, pack to before creation time
|
||||
continue
|
||||
raise KeyError(oid)
|
||||
|
||||
reachable[oid] = pos
|
||||
for oid in self.findrefs(pos):
|
||||
if oid not in reachable:
|
||||
todo.append(oid)
|
||||
|
||||
def findReachableFromFuture(self):
|
||||
# In this pass, the roots are positions of object revisions.
|
||||
# We add a pos to extra_roots when there is a backpointer to a
|
||||
# revision that was not current at the packtime. The
|
||||
# non-current revision could refer to objects that were
|
||||
# otherwise unreachable at the packtime.
|
||||
extra_roots = []
|
||||
|
||||
pos = self.packpos
|
||||
while pos < self.eof:
|
||||
th = self._read_txn_header(pos)
|
||||
self.checkTxn(th, pos)
|
||||
tpos = pos
|
||||
end = pos + th.tlen
|
||||
pos += th.headerlen()
|
||||
|
||||
while pos < end:
|
||||
dh = self._read_data_header(pos)
|
||||
self.checkData(th, tpos, dh, pos)
|
||||
|
||||
if dh.back and dh.back < self.packpos:
|
||||
if dh.oid in self.reachable:
|
||||
L = self.reach_ex.setdefault(dh.oid, [])
|
||||
if dh.back not in L:
|
||||
L.append(dh.back)
|
||||
extra_roots.append(dh.back)
|
||||
else:
|
||||
self.reachable[dh.oid] = dh.back
|
||||
|
||||
pos += dh.recordlen()
|
||||
|
||||
tlen = self._read_num(pos)
|
||||
if tlen != th.tlen:
|
||||
self.fail(pos, "redundant transaction length does not "
|
||||
"match initial transaction length: %d != %d",
|
||||
tlen, th.tlen)
|
||||
pos += 8
|
||||
|
||||
for pos in extra_roots:
|
||||
refs = self.findrefs(pos)
|
||||
self.findReachableAtPacktime(refs)
|
||||
|
||||
def findrefs(self, pos):
|
||||
"""Return a list of oids referenced as of packtime."""
|
||||
dh = self._read_data_header(pos)
|
||||
# Chase backpointers until we get to the record with the refs
|
||||
while dh.back:
|
||||
dh = self._read_data_header(dh.back)
|
||||
if dh.plen:
|
||||
return self.referencesf(self._file.read(dh.plen))
|
||||
else:
|
||||
return []
|
||||
|
||||
class FileStoragePacker(FileStorageFormatter):
|
||||
|
||||
# path is the storage file path.
|
||||
# stop is the pack time, as a TimeStamp.
|
||||
# current_size is the storage's _pos. All valid data at the start
|
||||
# lives before that offset (there may be a checkpoint transaction in
|
||||
# progress after it).
|
||||
|
||||
def __init__(self, storage, referencesf, stop, gc=True):
|
||||
self._storage = storage
|
||||
if storage.blob_dir:
|
||||
self.pack_blobs = True
|
||||
self.blob_removed = open(
|
||||
os.path.join(storage.blob_dir, '.removed'), 'wb')
|
||||
else:
|
||||
self.pack_blobs = False
|
||||
self.blob_removed = None
|
||||
|
||||
path = storage._file.name
|
||||
self._name = path
|
||||
# We open our own handle on the storage so that much of pack can
|
||||
# proceed in parallel. It's important to close this file at every
|
||||
# return point, else on Windows the caller won't be able to rename
|
||||
# or remove the storage file.
|
||||
self._file = open(path, "rb")
|
||||
self._path = path
|
||||
self._stop = stop
|
||||
self.locked = False
|
||||
self.file_end = storage.getSize()
|
||||
|
||||
self.gc = GC(self._file, self.file_end, self._stop, gc, referencesf)
|
||||
|
||||
# The packer needs to acquire the parent's commit lock
|
||||
# during the copying stage, so the two sets of lock acquire
|
||||
# and release methods are passed to the constructor.
|
||||
self._lock = storage._lock
|
||||
self._commit_lock = storage._commit_lock
|
||||
|
||||
# The packer will use several indexes.
|
||||
# index: oid -> pos
|
||||
# tindex: oid -> pos, for current txn
|
||||
# oid2tid: not used by the packer
|
||||
|
||||
self.index = ZODB.fsIndex.fsIndex()
|
||||
self.tindex = {}
|
||||
self.oid2tid = {}
|
||||
self.toid2tid = {}
|
||||
self.toid2tid_delete = {}
|
||||
|
||||
self._tfile = None
|
||||
|
||||
def close(self):
|
||||
self._file.close()
|
||||
if self._tfile is not None:
|
||||
self._tfile.close()
|
||||
if self.blob_removed is not None:
|
||||
self.blob_removed.close()
|
||||
|
||||
def pack(self):
|
||||
# Pack copies all data reachable at the pack time or later.
|
||||
#
|
||||
# Copying occurs in two phases. In the first phase, txns
|
||||
# before the pack time are copied if the contain any reachable
|
||||
# data. In the second phase, all txns after the pack time
|
||||
# are copied.
|
||||
#
|
||||
# Txn and data records contain pointers to previous records.
|
||||
# Because these pointers are stored as file offsets, they
|
||||
# must be updated when we copy data.
|
||||
|
||||
# TODO: Should add sanity checking to pack.
|
||||
|
||||
self.gc.findReachable()
|
||||
|
||||
def close_files_remove():
|
||||
# blank except: we might be in an IOError situation/handler
|
||||
# try our best, but don't fail
|
||||
try:
|
||||
self._tfile.close()
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
self._file.close()
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
os.remove(self._name + ".pack")
|
||||
except:
|
||||
pass
|
||||
if self.blob_removed is not None:
|
||||
self.blob_removed.close()
|
||||
|
||||
# Setup the destination file and copy the metadata.
|
||||
# TODO: rename from _tfile to something clearer.
|
||||
self._tfile = open(self._name + ".pack", "w+b")
|
||||
try:
|
||||
self._file.seek(0)
|
||||
self._tfile.write(self._file.read(self._metadata_size))
|
||||
|
||||
self._copier = PackCopier(self._tfile, self.index, self.tindex)
|
||||
|
||||
ipos, opos = self.copyToPacktime()
|
||||
except (OSError, IOError):
|
||||
# most probably ran out of disk space or some other IO error
|
||||
close_files_remove()
|
||||
raise # don't succeed silently
|
||||
|
||||
assert ipos == self.gc.packpos
|
||||
if ipos == opos:
|
||||
# pack didn't free any data. there's no point in continuing.
|
||||
close_files_remove()
|
||||
return None
|
||||
self._commit_lock.acquire()
|
||||
self.locked = True
|
||||
try:
|
||||
with self._lock:
|
||||
# Re-open the file in unbuffered mode.
|
||||
|
||||
# The main thread may write new transactions to the
|
||||
# file, which creates the possibility that we will
|
||||
# read a status 'c' transaction into the pack thread's
|
||||
# stdio buffer even though we're acquiring the commit
|
||||
# lock. Transactions can still be in progress
|
||||
# throughout much of packing, and are written to the
|
||||
# same physical file but via a distinct Python file
|
||||
# object. The code used to leave off the trailing 0
|
||||
# argument, and then on every platform except native
|
||||
# Windows it was observed that we could read stale
|
||||
# data from the tail end of the file.
|
||||
self._file.close() # else self.gc keeps the original
|
||||
# alive & open
|
||||
self._file = open(self._path, "rb", 0)
|
||||
self._file.seek(0, 2)
|
||||
self.file_end = self._file.tell()
|
||||
|
||||
if ipos < self.file_end:
|
||||
self.copyRest(ipos)
|
||||
|
||||
# OK, we've copied everything. Now we need to wrap things up.
|
||||
pos = self._tfile.tell()
|
||||
self._tfile.flush()
|
||||
self._tfile.close()
|
||||
self._file.close()
|
||||
if self.blob_removed is not None:
|
||||
self.blob_removed.close()
|
||||
|
||||
return pos
|
||||
except (OSError, IOError):
|
||||
# most probably ran out of disk space or some other IO error
|
||||
close_files_remove()
|
||||
if self.locked:
|
||||
self._commit_lock.release()
|
||||
raise # don't succeed silently
|
||||
except:
|
||||
if self.locked:
|
||||
self._commit_lock.release()
|
||||
raise
|
||||
|
||||
def copyToPacktime(self):
|
||||
offset = 0 # the amount of space freed by packing
|
||||
pos = self._metadata_size
|
||||
new_pos = pos
|
||||
|
||||
while pos < self.gc.packpos:
|
||||
th = self._read_txn_header(pos)
|
||||
new_tpos, pos = self.copyDataRecords(pos, th)
|
||||
|
||||
if new_tpos:
|
||||
new_pos = self._tfile.tell() + 8
|
||||
tlen = new_pos - new_tpos - 8
|
||||
# Update the transaction length
|
||||
self._tfile.seek(new_tpos + 8)
|
||||
self._tfile.write(p64(tlen))
|
||||
self._tfile.seek(new_pos - 8)
|
||||
self._tfile.write(p64(tlen))
|
||||
|
||||
|
||||
tlen = self._read_num(pos)
|
||||
if tlen != th.tlen:
|
||||
self.fail(pos, "redundant transaction length does not "
|
||||
"match initial transaction length: %d != %d",
|
||||
tlen, th.tlen)
|
||||
pos += 8
|
||||
|
||||
return pos, new_pos
|
||||
|
||||
def copyDataRecords(self, pos, th):
|
||||
"""Copy any current data records between pos and tend.
|
||||
|
||||
Returns position of txn header in output file and position
|
||||
of next record in the input file.
|
||||
|
||||
If any data records are copied, also write txn header (th).
|
||||
"""
|
||||
copy = 0
|
||||
new_tpos = 0
|
||||
tend = pos + th.tlen
|
||||
pos += th.headerlen()
|
||||
while pos < tend:
|
||||
h = self._read_data_header(pos)
|
||||
if not self.gc.isReachable(h.oid, pos):
|
||||
if self.pack_blobs:
|
||||
# We need to find out if this is a blob, so get the data:
|
||||
if h.plen:
|
||||
data = self._file.read(h.plen)
|
||||
else:
|
||||
data = self.fetchDataViaBackpointer(h.oid, h.back)
|
||||
if data and self._storage.is_blob_record(data):
|
||||
# We need to remove the blob record. Maybe we
|
||||
# need to remove oid:
|
||||
|
||||
# But first, we need to make sure the record
|
||||
# we're looking at isn't a dup of the current
|
||||
# record. There's a bug in ZEO blob support that causes
|
||||
# duplicate data records.
|
||||
rpos = self.gc.reachable.get(h.oid)
|
||||
is_dup = (rpos
|
||||
and self._read_data_header(rpos).tid == h.tid)
|
||||
if not is_dup:
|
||||
if h.oid not in self.gc.reachable:
|
||||
self.blob_removed.write(
|
||||
binascii.hexlify(h.oid)+b'\n')
|
||||
else:
|
||||
self.blob_removed.write(
|
||||
binascii.hexlify(h.oid+h.tid)+b'\n')
|
||||
|
||||
pos += h.recordlen()
|
||||
continue
|
||||
|
||||
pos += h.recordlen()
|
||||
|
||||
# If we are going to copy any data, we need to copy
|
||||
# the transaction header. Note that we will need to
|
||||
# patch up the transaction length when we are done.
|
||||
if not copy:
|
||||
th.status = "p"
|
||||
s = th.asString()
|
||||
new_tpos = self._tfile.tell()
|
||||
self._tfile.write(s)
|
||||
new_pos = new_tpos + len(s)
|
||||
copy = 1
|
||||
|
||||
if h.plen:
|
||||
data = self._file.read(h.plen)
|
||||
else:
|
||||
data = self.fetchDataViaBackpointer(h.oid, h.back)
|
||||
|
||||
self.writePackedDataRecord(h, data, new_tpos)
|
||||
new_pos = self._tfile.tell()
|
||||
|
||||
return new_tpos, pos
|
||||
|
||||
def fetchDataViaBackpointer(self, oid, back):
|
||||
"""Return the data for oid via backpointer back
|
||||
|
||||
If `back` is 0 or ultimately resolves to 0, return None.
|
||||
In this case, the transaction undoes the object
|
||||
creation.
|
||||
"""
|
||||
if back == 0:
|
||||
return None
|
||||
data, tid = self._loadBackTxn(oid, back, 0)
|
||||
return data
|
||||
|
||||
def writePackedDataRecord(self, h, data, new_tpos):
|
||||
# Update the header to reflect current information, then write
|
||||
# it to the output file.
|
||||
if data is None:
|
||||
data = b''
|
||||
h.prev = 0
|
||||
h.back = 0
|
||||
h.plen = len(data)
|
||||
h.tloc = new_tpos
|
||||
pos = self._tfile.tell()
|
||||
self.index[h.oid] = pos
|
||||
self._tfile.write(h.asString())
|
||||
self._tfile.write(data)
|
||||
if not data:
|
||||
# Packed records never have backpointers (?).
|
||||
# If there is no data, write a z64 backpointer.
|
||||
# This is a George Bailey event.
|
||||
self._tfile.write(z64)
|
||||
|
||||
def copyRest(self, ipos):
|
||||
# After the pack time, all data records are copied.
|
||||
# Copy one txn at a time, using copy() for data.
|
||||
|
||||
try:
|
||||
while 1:
|
||||
ipos = self.copyOne(ipos)
|
||||
except CorruptedDataError as err:
|
||||
# The last call to copyOne() will raise
|
||||
# CorruptedDataError, because it will attempt to read past
|
||||
# the end of the file. Double-check that the exception
|
||||
# occurred for this reason.
|
||||
self._file.seek(0, 2)
|
||||
endpos = self._file.tell()
|
||||
if endpos != err.pos:
|
||||
raise
|
||||
|
||||
def copyOne(self, ipos):
|
||||
# The call below will raise CorruptedDataError at EOF.
|
||||
th = self._read_txn_header(ipos)
|
||||
# Release commit lock while writing to pack file
|
||||
self._commit_lock.release()
|
||||
self.locked = False
|
||||
pos = self._tfile.tell()
|
||||
self._copier.setTxnPos(pos)
|
||||
self._tfile.write(th.asString())
|
||||
tend = ipos + th.tlen
|
||||
ipos += th.headerlen()
|
||||
|
||||
while ipos < tend:
|
||||
h = self._read_data_header(ipos)
|
||||
ipos += h.recordlen()
|
||||
prev_txn = None
|
||||
if h.plen:
|
||||
data = self._file.read(h.plen)
|
||||
else:
|
||||
data = self.fetchDataViaBackpointer(h.oid, h.back)
|
||||
if h.back:
|
||||
prev_txn = self.getTxnFromData(h.oid, h.back)
|
||||
|
||||
self._copier.copy(h.oid, h.tid, data, prev_txn,
|
||||
pos, self._tfile.tell())
|
||||
|
||||
tlen = self._tfile.tell() - pos
|
||||
assert tlen == th.tlen
|
||||
self._tfile.write(p64(tlen))
|
||||
ipos += 8
|
||||
|
||||
self.index.update(self.tindex)
|
||||
self.tindex.clear()
|
||||
self._commit_lock.acquire()
|
||||
self.locked = True
|
||||
return ipos
|
||||
@@ -0,0 +1,77 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) Zope Corporation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
import zope.interface
|
||||
|
||||
class IFileStoragePacker(zope.interface.Interface):
|
||||
|
||||
def __call__(storage, referencesf, stop, gc):
|
||||
r"""Pack the file storage into a new file
|
||||
|
||||
:param FileStorage storage: The storage object to be packed
|
||||
:param callable referencesf: A function that extracts object
|
||||
references from a pickle bytes string. This is usually
|
||||
``ZODB.serialize.referencesf``.
|
||||
:param bytes stop: A transaction id representing the time at
|
||||
which to stop packing.
|
||||
:param bool gc: A flag indicating whether garbage collection
|
||||
should be performed.
|
||||
|
||||
The new file will have the same name as the old file with
|
||||
``.pack`` appended. (The packer can get the old file name via
|
||||
storage._file.name.) If blobs are supported, if the storages
|
||||
blob_dir attribute is not None or empty, then a .removed file
|
||||
must be created in the blob directory. This file contains records of
|
||||
the form::
|
||||
|
||||
(oid+serial).encode('hex')+'\n'
|
||||
|
||||
or, of the form::
|
||||
|
||||
oid.encode('hex')+'\n'
|
||||
|
||||
If packing is unnecessary, or would not change the file, then
|
||||
no pack or removed files are created None is returned,
|
||||
otherwise a tuple is returned with:
|
||||
|
||||
- the size of the packed file, and
|
||||
|
||||
- the packed index
|
||||
|
||||
If and only if packing was necessary (non-None) and there was
|
||||
no error, then the commit lock must be acquired. In addition,
|
||||
it is up to FileStorage to:
|
||||
|
||||
- Rename the .pack file, and
|
||||
|
||||
- process the blob_dir/.removed file by removing the blobs
|
||||
corresponding to the file records.
|
||||
"""
|
||||
|
||||
class IFileStorage(zope.interface.Interface):
|
||||
|
||||
packer = zope.interface.Attribute(
|
||||
"The IFileStoragePacker to be used for packing."
|
||||
)
|
||||
|
||||
_file = zope.interface.Attribute(
|
||||
"The file object used to access the underlying data."
|
||||
)
|
||||
|
||||
_lock = zope.interface.Attribute(
|
||||
"The storage lock."
|
||||
)
|
||||
|
||||
_commit_lock = zope.interface.Attribute(
|
||||
"The storage commit lock."
|
||||
)
|
||||
@@ -0,0 +1,182 @@
|
||||
FileStorage-specific iterator tests
|
||||
===================================
|
||||
|
||||
The FileStorage iterator has some special features that deserve some
|
||||
special tests.
|
||||
|
||||
We'll make some assertions about time, so we'll take it over:
|
||||
|
||||
>>> now = 1229959248
|
||||
>>> def faux_time():
|
||||
... global now
|
||||
... now += 0.1
|
||||
... return now
|
||||
>>> import time
|
||||
>>> time_time = time.time
|
||||
>>> if isinstance(time,type):
|
||||
... time.time = staticmethod(faux_time) # Jython
|
||||
... else:
|
||||
... time.time = faux_time
|
||||
|
||||
Commit a bunch of transactions:
|
||||
|
||||
>>> import ZODB.FileStorage, transaction
|
||||
>>> db = ZODB.DB('data.fs')
|
||||
>>> tids = [db.storage.lastTransaction()]
|
||||
>>> poss = [db.storage._pos]
|
||||
>>> conn = db.open()
|
||||
>>> for i in range(100):
|
||||
... conn.root()[i] = conn.root().__class__()
|
||||
... transaction.commit()
|
||||
... tids.append(db.storage.lastTransaction())
|
||||
... poss.append(db.storage._pos)
|
||||
|
||||
Deciding where to start
|
||||
-----------------------
|
||||
|
||||
By default, we start at the beginning:
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs')
|
||||
>>> it.next().tid == tids[0]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
The file iterator has an optimization to deal with large files. It
|
||||
can serarch from either the front or the back of the file, depending
|
||||
on the starting transaction given. To see this, we'll turn on debug
|
||||
logging:
|
||||
|
||||
>>> import logging, sys
|
||||
>>> old_log_level = logging.getLogger().getEffectiveLevel()
|
||||
>>> logging.getLogger().setLevel(logging.DEBUG)
|
||||
>>> handler = logging.StreamHandler(sys.stdout)
|
||||
>>> logging.getLogger().addHandler(handler)
|
||||
|
||||
If we specify a start transaction, we'll scan forward or backward, as
|
||||
seems best and set the next record to that:
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[0])
|
||||
>>> it.next().tid == tids[0]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[1])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan forward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[1]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[30])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan forward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[30]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[70])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan backward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[70]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[-2])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan backward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[-2]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[-1])
|
||||
>>> it.next().tid == tids[-1]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
We can also supply a file position. This can speed up finding the
|
||||
starting point, or just pick up where another iterator left off:
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', pos=poss[50])
|
||||
>>> it.next().tid == tids[51]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[0], pos=4)
|
||||
>>> it.next().tid == tids[0]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[-1], pos=poss[-2])
|
||||
>>> it.next().tid == tids[-1]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[50], pos=poss[50])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan backward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[50]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[49], pos=poss[50])
|
||||
... # doctest: +ELLIPSIS
|
||||
Scan backward data.fs:<OFFSET> looking for ...
|
||||
>>> it.next().tid == tids[49]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', tids[51], pos=poss[50])
|
||||
>>> it.next().tid == tids[51]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
>>> logging.getLogger().setLevel(old_log_level)
|
||||
>>> logging.getLogger().removeHandler(handler)
|
||||
|
||||
|
||||
If a starting transaction is before the first transaction in the file,
|
||||
then the first transaction is returned.
|
||||
|
||||
>>> from ZODB.utils import p64, u64
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', p64(u64(tids[0])-1))
|
||||
>>> it.next().tid == tids[0]
|
||||
True
|
||||
|
||||
>>> it.close()
|
||||
|
||||
If it is after the last transaction, then iteration be empty:
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', p64(u64(tids[-1])+1))
|
||||
>>> list(it)
|
||||
[]
|
||||
|
||||
>>> it.close()
|
||||
|
||||
Even if we write more transactions:
|
||||
|
||||
>>> it = ZODB.FileStorage.FileIterator('data.fs', p64(u64(tids[-1])+1))
|
||||
>>> for i in range(10):
|
||||
... conn.root()[i] = conn.root().__class__()
|
||||
... transaction.commit()
|
||||
>>> list(it)
|
||||
[]
|
||||
|
||||
>>> it.close()
|
||||
|
||||
.. Cleanup
|
||||
|
||||
>>> time.time = time_time
|
||||
>>> db.close()
|
||||
@@ -0,0 +1,322 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) Zope Corporation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
import doctest
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import transaction
|
||||
import unittest
|
||||
import ZODB.blob
|
||||
import ZODB.FileStorage
|
||||
import ZODB.tests.util
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from zope.testing import renormalizing
|
||||
|
||||
checker = renormalizing.RENormalizing([
|
||||
# Python 3 bytes add a "b".
|
||||
(re.compile("b('.*?')"), r"\1"),
|
||||
# Python 3 adds module name to exceptions.
|
||||
(re.compile("ZODB.POSException.POSKeyError"), r"POSKeyError"),
|
||||
(re.compile("ZODB.FileStorage.FileStorage.FileStorageQuotaError"),
|
||||
"FileStorageQuotaError"),
|
||||
(re.compile('data.fs:[0-9]+'), 'data.fs:<OFFSET>'),
|
||||
])
|
||||
|
||||
def pack_keep_old():
|
||||
"""Should a copy of the database be kept?
|
||||
|
||||
The pack_keep_old constructor argument controls whether a .old file (and .old
|
||||
directory for blobs is kept.)
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> import ZODB.blob
|
||||
>>> conn.root()[1] = ZODB.blob.Blob()
|
||||
>>> with conn.root()[1].open('w') as file:
|
||||
... _ = file.write(b'some data')
|
||||
>>> conn.root()[2] = ZODB.blob.Blob()
|
||||
>>> with conn.root()[2].open('w') as file:
|
||||
... _ = file.write(b'some data')
|
||||
>>> transaction.commit()
|
||||
>>> with conn.root()[1].open('w') as file:
|
||||
... _ = file.write(b'some other data')
|
||||
>>> del conn.root()[2]
|
||||
>>> transaction.commit()
|
||||
>>> old_size = os.stat('data.fs').st_size
|
||||
>>> def get_blob_size(d):
|
||||
... result = 0
|
||||
... for path, dirs, file_names in os.walk(d):
|
||||
... for file_name in file_names:
|
||||
... result += os.stat(os.path.join(path, file_name)).st_size
|
||||
... return result
|
||||
>>> blob_size = get_blob_size('blobs')
|
||||
|
||||
>>> db.pack(time.time()+1)
|
||||
>>> packed_size = os.stat('data.fs').st_size
|
||||
>>> packed_size < old_size
|
||||
True
|
||||
>>> os.stat('data.fs.old').st_size == old_size
|
||||
True
|
||||
|
||||
>>> packed_blob_size = get_blob_size('blobs')
|
||||
>>> packed_blob_size < blob_size
|
||||
True
|
||||
>>> get_blob_size('blobs.old') == blob_size
|
||||
True
|
||||
>>> db.close()
|
||||
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs', blob_dir='blobs',
|
||||
... create=True, pack_keep_old=False)
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1] = ZODB.blob.Blob()
|
||||
>>> with conn.root()[1].open('w') as file:
|
||||
... _ = file.write(b'some data')
|
||||
>>> conn.root()[2] = ZODB.blob.Blob()
|
||||
>>> with conn.root()[2].open('w') as file:
|
||||
... _ = file.write(b'some data')
|
||||
>>> transaction.commit()
|
||||
>>> with conn.root()[1].open('w') as file:
|
||||
... _ = file.write(b'some other data')
|
||||
>>> del conn.root()[2]
|
||||
>>> transaction.commit()
|
||||
|
||||
>>> db.pack(time.time()+1)
|
||||
>>> os.stat('data.fs').st_size == packed_size
|
||||
True
|
||||
>>> os.path.exists('data.fs.old')
|
||||
False
|
||||
>>> get_blob_size('blobs') == packed_blob_size
|
||||
True
|
||||
>>> os.path.exists('blobs.old')
|
||||
False
|
||||
>>> db.close()
|
||||
"""
|
||||
|
||||
def pack_with_repeated_blob_records():
|
||||
"""
|
||||
There is a bug in ZEO that causes duplicate bloc database records
|
||||
to be written in a blob store operation. (Maybe this has been
|
||||
fixed by the time you read this, but there might still be
|
||||
transactions in the wild that have duplicate records.
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('t', blob_dir='bobs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1] = ZODB.blob.Blob()
|
||||
>>> transaction.commit()
|
||||
>>> tm = transaction.TransactionManager()
|
||||
>>> oid = conn.root()[1]._p_oid
|
||||
>>> from ZODB.utils import load_current
|
||||
>>> blob_record, oldserial = load_current(fs, oid)
|
||||
|
||||
Now, create a transaction with multiple saves:
|
||||
|
||||
>>> trans = TransactionMetaData()
|
||||
>>> fs.tpc_begin(trans)
|
||||
>>> with open('ablob', 'w') as file:
|
||||
... _ = file.write('some data')
|
||||
>>> fs.store(oid, oldserial, blob_record, '', trans)
|
||||
>>> fs.storeBlob(oid, oldserial, blob_record, 'ablob', '', trans)
|
||||
>>> _ = fs.tpc_vote(trans)
|
||||
>>> _ = fs.tpc_finish(trans)
|
||||
|
||||
>>> time.sleep(.01)
|
||||
>>> db.pack()
|
||||
|
||||
>>> conn.sync()
|
||||
>>> with conn.root()[1].open() as fp: fp.read()
|
||||
'some data'
|
||||
|
||||
>>> db.close()
|
||||
"""
|
||||
|
||||
def _save_index():
|
||||
"""
|
||||
|
||||
_save_index can fail for large indexes.
|
||||
|
||||
>>> import ZODB.utils
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs')
|
||||
|
||||
>>> t = TransactionMetaData()
|
||||
>>> fs.tpc_begin(t)
|
||||
>>> oid = 0
|
||||
>>> for i in range(5000):
|
||||
... oid += (1<<16)
|
||||
... fs.store(ZODB.utils.p64(oid), ZODB.utils.z64, b'x', '', t)
|
||||
>>> _ = fs.tpc_vote(t)
|
||||
>>> _ = fs.tpc_finish(t)
|
||||
|
||||
>>> import sys
|
||||
>>> old_limit = sys.getrecursionlimit()
|
||||
>>> sys.setrecursionlimit(50)
|
||||
>>> fs._save_index()
|
||||
|
||||
Make sure we can restore:
|
||||
|
||||
>>> import logging
|
||||
>>> handler = logging.StreamHandler(sys.stdout)
|
||||
>>> logger = logging.getLogger('ZODB.FileStorage')
|
||||
>>> logger.setLevel(logging.DEBUG)
|
||||
>>> logger.addHandler(handler)
|
||||
>>> index, pos, tid = fs._restore_index()
|
||||
>>> index.items() == fs._index.items()
|
||||
True
|
||||
>>> pos, tid = fs._pos, fs._tid
|
||||
|
||||
cleanup
|
||||
|
||||
>>> fs.close()
|
||||
>>> logger.setLevel(logging.NOTSET)
|
||||
>>> logger.removeHandler(handler)
|
||||
>>> sys.setrecursionlimit(old_limit)
|
||||
|
||||
"""
|
||||
|
||||
def pack_disk_full_copyToPacktime():
|
||||
"""Recover from a disk full situation by removing the `.pack` file
|
||||
|
||||
`copyToPacktime` fails
|
||||
|
||||
Add some data
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1] = 'foobar'
|
||||
>>> transaction.commit()
|
||||
|
||||
patch `copyToPacktime` to fail
|
||||
|
||||
>>> from ZODB.FileStorage import fspack
|
||||
>>> save_copyToPacktime = fspack.FileStoragePacker.copyToPacktime
|
||||
|
||||
>>> def failing_copyToPacktime(self):
|
||||
... self._tfile.write(b'somejunkdata')
|
||||
... raise OSError("No space left on device")
|
||||
|
||||
>>> fspack.FileStoragePacker.copyToPacktime = failing_copyToPacktime
|
||||
|
||||
pack -- it still raises `OSError`
|
||||
|
||||
>>> db.pack(time.time()+1)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
OSError: No space left on device
|
||||
|
||||
`data.fs.pack` must not exist
|
||||
|
||||
>>> os.path.exists('data.fs.pack')
|
||||
False
|
||||
|
||||
undo patching
|
||||
|
||||
>>> fspack.FileStoragePacker.copyToPacktime = save_copyToPacktime
|
||||
|
||||
>>> db.close()
|
||||
|
||||
check the data we added
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1]
|
||||
'foobar'
|
||||
>>> db.close()
|
||||
"""
|
||||
|
||||
def pack_disk_full_copyRest():
|
||||
"""Recover from a disk full situation by removing the `.pack` file
|
||||
|
||||
`copyRest` fails
|
||||
|
||||
Add some data
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1] = 'foobar'
|
||||
>>> transaction.commit()
|
||||
|
||||
patch `copyToPacktime` to add one more transaction
|
||||
|
||||
>>> from ZODB.FileStorage import fspack
|
||||
>>> save_copyToPacktime = fspack.FileStoragePacker.copyToPacktime
|
||||
|
||||
>>> def patched_copyToPacktime(self):
|
||||
... res = save_copyToPacktime(self)
|
||||
... conn2 = db.open()
|
||||
... conn2.root()[2] = 'another bar'
|
||||
... transaction.commit()
|
||||
... return res
|
||||
|
||||
>>> fspack.FileStoragePacker.copyToPacktime = patched_copyToPacktime
|
||||
|
||||
patch `copyRest` to fail
|
||||
|
||||
>>> save_copyRest = fspack.FileStoragePacker.copyRest
|
||||
|
||||
>>> def failing_copyRest(self, ipos):
|
||||
... self._tfile.write(b'somejunkdata')
|
||||
... raise OSError("No space left on device")
|
||||
|
||||
>>> fspack.FileStoragePacker.copyRest = failing_copyRest
|
||||
|
||||
pack -- it still raises `OSError`
|
||||
|
||||
>>> db.pack(time.time()+1)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
OSError: No space left on device
|
||||
|
||||
`data.fs.pack` must not exist
|
||||
|
||||
>>> os.path.exists('data.fs.pack')
|
||||
False
|
||||
|
||||
undo patching
|
||||
|
||||
>>> fspack.FileStoragePacker.copyToPacktime = save_copyToPacktime
|
||||
>>> fspack.FileStoragePacker.copyRest = save_copyRest
|
||||
|
||||
>>> db.close()
|
||||
|
||||
check the data we added
|
||||
|
||||
>>> fs = ZODB.FileStorage.FileStorage('data.fs')
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[1]
|
||||
'foobar'
|
||||
>>> conn.root()[2]
|
||||
'another bar'
|
||||
>>> db.close()
|
||||
"""
|
||||
|
||||
def test_suite():
|
||||
return unittest.TestSuite((
|
||||
doctest.DocFileSuite(
|
||||
'zconfig.txt',
|
||||
'iterator.test',
|
||||
setUp=ZODB.tests.util.setUp,
|
||||
tearDown=ZODB.tests.util.tearDown,
|
||||
checker=checker),
|
||||
doctest.DocTestSuite(
|
||||
setUp=ZODB.tests.util.setUp,
|
||||
tearDown=ZODB.tests.util.tearDown,
|
||||
checker=checker),
|
||||
))
|
||||
@@ -0,0 +1,209 @@
|
||||
Defining FileStorages using ZConfig
|
||||
===================================
|
||||
|
||||
ZODB provides support for defining many storages, including
|
||||
FileStorages, using ZConfig. To define a FileStorage, you use a
|
||||
filestorage section, and define a path:
|
||||
|
||||
>>> import ZODB.config
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> fs._file.name
|
||||
'my.fs'
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
There are a number of options we can provide:
|
||||
|
||||
blob-dir
|
||||
If supplied, the file storage will provide blob support and this
|
||||
is the name of a directory to hold blob data. The directory will
|
||||
be created if it doeesn't exist. If no value (or an empty value)
|
||||
is provided, then no blob support will be provided. (You can still
|
||||
use a BlobStorage to provide blob support.)
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... blob-dir blobs
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> fs._file.name
|
||||
'my.fs'
|
||||
>>> import os
|
||||
>>> os.path.basename(fs.blob_dir)
|
||||
'blobs'
|
||||
|
||||
create
|
||||
Flag that indicates whether the storage should be truncated if
|
||||
it already exists.
|
||||
|
||||
To demonstrate this, we'll first write some data:
|
||||
|
||||
>>> db = ZODB.DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> import ZODB.blob, transaction
|
||||
>>> conn.root()[1] = ZODB.blob.Blob()
|
||||
>>> transaction.commit()
|
||||
>>> db.close()
|
||||
|
||||
Then reopen with the create option:
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... blob-dir blobs
|
||||
... create true
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
Because the file was truncated, we no-longer have object 0:
|
||||
|
||||
>>> fs.load(b'\0'*8)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: 0x00
|
||||
|
||||
>>> sorted(os.listdir('blobs'))
|
||||
['.layout', 'tmp']
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
read-only
|
||||
If true, only reads may be executed against the storage. Note
|
||||
that the "pack" operation is not considered a write operation
|
||||
and is still allowed on a read-only filestorage.
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... read-only true
|
||||
... </filestorage>
|
||||
... """)
|
||||
>>> fs.isReadOnly()
|
||||
True
|
||||
>>> fs.close()
|
||||
|
||||
quota
|
||||
Maximum allowed size of the storage file. Operations which
|
||||
would cause the size of the storage to exceed the quota will
|
||||
result in a ZODB.FileStorage.FileStorageQuotaError being
|
||||
raised.
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... quota 10
|
||||
... </filestorage>
|
||||
... """)
|
||||
>>> db = ZODB.DB(fs) # writes object 0
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
FileStorageQuotaError: The storage quota has been exceeded.
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
packer
|
||||
The dotten name (dotten module name and object name) of a
|
||||
packer object. This is used to provide an alternative pack
|
||||
implementation.
|
||||
|
||||
To demonstrate this, we'll create a null packer that just prints
|
||||
some information about it's arguments:
|
||||
|
||||
>>> import six
|
||||
>>> def packer(storage, referencesf, stop, gc):
|
||||
... six.print_(referencesf, storage is fs, gc, storage.pack_keep_old)
|
||||
>>> ZODB.FileStorage.config_demo_printing_packer = packer
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... packer ZODB.FileStorage.config_demo_printing_packer
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> import time
|
||||
>>> db = ZODB.DB(fs) # writes object 0
|
||||
>>> fs.pack(time.time(), 42)
|
||||
42 True True True
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
If the packer contains a ':', then the text after the first ':' is
|
||||
interpreted as an expression. This is handy to pass limited
|
||||
configuration information to the packer:
|
||||
|
||||
>>> def packer_factory(name):
|
||||
... def packer(storage, referencesf, stop, gc):
|
||||
... six.print_(repr(name), referencesf, storage is fs,
|
||||
... gc, storage.pack_keep_old)
|
||||
... return packer
|
||||
>>> ZODB.FileStorage.config_demo_printing_packer_factory = packer_factory
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... packer ZODB.FileStorage:config_demo_printing_packer_factory('bob ')
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> import time
|
||||
>>> db = ZODB.DB(fs) # writes object 0
|
||||
>>> fs.pack(time.time(), 42)
|
||||
'bob ' 42 True True True
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
|
||||
|
||||
pack-gc
|
||||
If false, then no garbage collection will be performed when
|
||||
packing. This can make packing go much faster and can avoid
|
||||
problems when objects are referenced only from other
|
||||
databases.
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... packer ZODB.FileStorage.config_demo_printing_packer
|
||||
... pack-gc false
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> fs.pack(time.time(), 42)
|
||||
42 True False True
|
||||
|
||||
Note that if we pass the gc option to pack, then this will
|
||||
override the value set in the configuration:
|
||||
|
||||
>>> fs.pack(time.time(), 42, gc=True)
|
||||
42 True True True
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
pack-keep-old
|
||||
If false, then old files aren't kept when packing
|
||||
|
||||
>>> fs = ZODB.config.storageFromString("""
|
||||
... <filestorage>
|
||||
... path my.fs
|
||||
... packer ZODB.FileStorage.config_demo_printing_packer
|
||||
... pack-keep-old false
|
||||
... </filestorage>
|
||||
... """)
|
||||
|
||||
>>> fs.pack(time.time(), 42)
|
||||
42 True True False
|
||||
|
||||
>>> fs.close()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""A simple in-memory mapping-based ZODB storage
|
||||
|
||||
This storage provides an example implementation of a fairly full
|
||||
storage without distracting storage details.
|
||||
"""
|
||||
|
||||
import BTrees
|
||||
import time
|
||||
import ZODB.BaseStorage
|
||||
import ZODB.interfaces
|
||||
import ZODB.POSException
|
||||
import ZODB.TimeStamp
|
||||
import ZODB.utils
|
||||
import zope.interface
|
||||
|
||||
|
||||
@zope.interface.implementer(
|
||||
ZODB.interfaces.IStorage,
|
||||
ZODB.interfaces.IStorageIteration,
|
||||
)
|
||||
class MappingStorage(object):
|
||||
"""In-memory storage implementation
|
||||
|
||||
Note that this implementation is somewhat naive and inefficient
|
||||
with regard to locking. Its implementation is primarily meant to
|
||||
be a simple illustration of storage implementation. It's also
|
||||
useful for testing and exploration where scalability and efficiency
|
||||
are unimportant.
|
||||
"""
|
||||
|
||||
def __init__(self, name='MappingStorage'):
|
||||
"""Create a mapping storage
|
||||
|
||||
The name parameter is used by the
|
||||
:meth:`~ZODB.interfaces.IStorage.getName` and
|
||||
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
|
||||
"""
|
||||
self.__name__ = name
|
||||
self._data = {} # {oid->{tid->pickle}}
|
||||
self._transactions = BTrees.OOBTree.OOBTree() # {tid->TransactionRecord}
|
||||
self._ltid = ZODB.utils.z64
|
||||
self._last_pack = None
|
||||
self._lock = ZODB.utils.RLock()
|
||||
self._commit_lock = ZODB.utils.Lock()
|
||||
self._opened = True
|
||||
self._transaction = None
|
||||
self._oid = 0
|
||||
|
||||
######################################################################
|
||||
# Preconditions:
|
||||
|
||||
def opened(self):
|
||||
"""The storage is open
|
||||
"""
|
||||
return self._opened
|
||||
|
||||
def not_in_transaction(self):
|
||||
"""The storage is not committing a transaction
|
||||
"""
|
||||
return self._transaction is None
|
||||
|
||||
#
|
||||
######################################################################
|
||||
|
||||
# testing framework (lame)
|
||||
def cleanup(self):
|
||||
pass
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked
|
||||
def close(self):
|
||||
self._opened = False
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def getName(self):
|
||||
return self.__name__
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def getSize(self):
|
||||
size = 0
|
||||
for oid, tid_data in self._data.items():
|
||||
size += 50
|
||||
for tid, pickle in tid_data.items():
|
||||
size += 100+len(pickle)
|
||||
return size
|
||||
|
||||
# ZEO.interfaces.IServeable
|
||||
@ZODB.utils.locked(opened)
|
||||
def getTid(self, oid):
|
||||
tid_data = self._data.get(oid)
|
||||
if tid_data:
|
||||
return tid_data.maxKey()
|
||||
raise ZODB.POSException.POSKeyError(oid)
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def history(self, oid, size=1):
|
||||
tid_data = self._data.get(oid)
|
||||
if not tid_data:
|
||||
raise ZODB.POSException.POSKeyError(oid)
|
||||
|
||||
tids = tid_data.keys()[-size:]
|
||||
tids.reverse()
|
||||
return [
|
||||
dict(
|
||||
time = ZODB.TimeStamp.TimeStamp(tid).timeTime(),
|
||||
tid = tid,
|
||||
serial = tid,
|
||||
user_name = self._transactions[tid].user,
|
||||
description = self._transactions[tid].description,
|
||||
extension = self._transactions[tid].extension,
|
||||
size = len(tid_data[tid])
|
||||
)
|
||||
for tid in tids]
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def isReadOnly(self):
|
||||
return False
|
||||
|
||||
# ZODB.interfaces.IStorageIteration
|
||||
def iterator(self, start=None, end=None):
|
||||
for transaction_record in self._transactions.values(start, end):
|
||||
yield transaction_record
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def lastTransaction(self):
|
||||
return self._ltid
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def __len__(self):
|
||||
return len(self._data)
|
||||
|
||||
load = ZODB.utils.load_current
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def loadBefore(self, oid, tid):
|
||||
tid_data = self._data.get(oid)
|
||||
if tid_data:
|
||||
before = ZODB.utils.u64(tid)
|
||||
if not before:
|
||||
return None
|
||||
before = ZODB.utils.p64(before-1)
|
||||
tids_before = tid_data.keys(None, before)
|
||||
if tids_before:
|
||||
tids_after = tid_data.keys(tid, None)
|
||||
tid = tids_before[-1]
|
||||
return (tid_data[tid], tid,
|
||||
(tids_after and tids_after[0] or None)
|
||||
)
|
||||
else:
|
||||
raise ZODB.POSException.POSKeyError(oid)
|
||||
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def loadSerial(self, oid, serial):
|
||||
tid_data = self._data.get(oid)
|
||||
if tid_data:
|
||||
try:
|
||||
return tid_data[serial]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
raise ZODB.POSException.POSKeyError(oid, serial)
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def new_oid(self):
|
||||
self._oid += 1
|
||||
return ZODB.utils.p64(self._oid)
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def pack(self, t, referencesf, gc=True):
|
||||
if not self._data:
|
||||
return
|
||||
|
||||
stop = ZODB.TimeStamp.TimeStamp(*time.gmtime(t)[:5]+(t%60,)).raw()
|
||||
if self._last_pack is not None and self._last_pack >= stop:
|
||||
if self._last_pack == stop:
|
||||
return
|
||||
raise ValueError("Already packed to a later time")
|
||||
|
||||
self._last_pack = stop
|
||||
transactions = self._transactions
|
||||
|
||||
# Step 1, remove old non-current records
|
||||
for oid, tid_data in self._data.items():
|
||||
tids_to_remove = tid_data.keys(None, stop)
|
||||
if tids_to_remove:
|
||||
tids_to_remove.pop() # Keep the last, if any
|
||||
|
||||
if tids_to_remove:
|
||||
for tid in tids_to_remove:
|
||||
del tid_data[tid]
|
||||
if transactions[tid].pack(oid):
|
||||
del transactions[tid]
|
||||
|
||||
if gc:
|
||||
# Step 2, GC. A simple sweep+copy
|
||||
new_data = BTrees.OOBTree.OOBTree()
|
||||
to_copy = set([ZODB.utils.z64])
|
||||
while to_copy:
|
||||
oid = to_copy.pop()
|
||||
tid_data = self._data.pop(oid)
|
||||
new_data[oid] = tid_data
|
||||
for pickle in tid_data.values():
|
||||
for oid in referencesf(pickle):
|
||||
if oid in new_data:
|
||||
continue
|
||||
to_copy.add(oid)
|
||||
|
||||
# Remove left over data from transactions
|
||||
for oid, tid_data in self._data.items():
|
||||
for tid in tid_data:
|
||||
if transactions[tid].pack(oid):
|
||||
del transactions[tid]
|
||||
|
||||
self._data.clear()
|
||||
self._data.update(new_data)
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def registerDB(self, db):
|
||||
pass
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def sortKey(self):
|
||||
return self.__name__
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def store(self, oid, serial, data, version, transaction):
|
||||
assert not version, "Versions are not supported"
|
||||
if transaction is not self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(self, transaction)
|
||||
|
||||
old_tid = None
|
||||
tid_data = self._data.get(oid)
|
||||
if tid_data:
|
||||
old_tid = tid_data.maxKey()
|
||||
if serial != old_tid:
|
||||
raise ZODB.POSException.ConflictError(
|
||||
oid=oid, serials=(old_tid, serial), data=data)
|
||||
|
||||
self._tdata[oid] = data
|
||||
|
||||
checkCurrentSerialInTransaction = (
|
||||
ZODB.BaseStorage.checkCurrentSerialInTransaction)
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def tpc_abort(self, transaction):
|
||||
if transaction is not self._transaction:
|
||||
return
|
||||
self._transaction = None
|
||||
self._commit_lock.release()
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def tpc_begin(self, transaction, tid=None):
|
||||
with self._lock:
|
||||
|
||||
ZODB.utils.check_precondition(self.opened)
|
||||
|
||||
# The tid argument exists to support testing.
|
||||
if transaction is self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"Duplicate tpc_begin calls for same transaction")
|
||||
|
||||
self._commit_lock.acquire()
|
||||
|
||||
with self._lock:
|
||||
self._transaction = transaction
|
||||
self._tdata = {}
|
||||
if tid is None:
|
||||
if self._transactions:
|
||||
old_tid = self._transactions.maxKey()
|
||||
else:
|
||||
old_tid = None
|
||||
tid = ZODB.utils.newTid(old_tid)
|
||||
self._tid = tid
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
@ZODB.utils.locked(opened)
|
||||
def tpc_finish(self, transaction, func = lambda tid: None):
|
||||
if (transaction is not self._transaction):
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"tpc_finish called with wrong transaction")
|
||||
|
||||
tid = self._tid
|
||||
func(tid)
|
||||
|
||||
tdata = self._tdata
|
||||
for oid in tdata:
|
||||
tid_data = self._data.get(oid)
|
||||
if tid_data is None:
|
||||
tid_data = BTrees.OOBTree.OOBucket()
|
||||
self._data[oid] = tid_data
|
||||
tid_data[tid] = tdata[oid]
|
||||
|
||||
self._ltid = tid
|
||||
self._transactions[tid] = TransactionRecord(tid, transaction, tdata)
|
||||
self._transaction = None
|
||||
del self._tdata
|
||||
self._commit_lock.release()
|
||||
return tid
|
||||
|
||||
# ZEO.interfaces.IServeable
|
||||
@ZODB.utils.locked(opened)
|
||||
def tpc_transaction(self):
|
||||
return self._transaction
|
||||
|
||||
# ZODB.interfaces.IStorage
|
||||
def tpc_vote(self, transaction):
|
||||
if transaction is not self._transaction:
|
||||
raise ZODB.POSException.StorageTransactionError(
|
||||
"tpc_vote called with wrong transaction")
|
||||
|
||||
class TransactionRecord(object):
|
||||
|
||||
status = ' '
|
||||
|
||||
def __init__(self, tid, transaction, data):
|
||||
self.tid = tid
|
||||
self.user = transaction.user
|
||||
self.description = transaction.description
|
||||
extension = transaction.extension
|
||||
self.extension = extension
|
||||
self.data = data
|
||||
|
||||
_extension = property(lambda self: self.extension,
|
||||
lambda self, v: setattr(self, 'extension', v),
|
||||
)
|
||||
|
||||
def __iter__(self):
|
||||
for oid, data in self.data.items():
|
||||
yield DataRecord(oid, self.tid, data)
|
||||
|
||||
def pack(self, oid):
|
||||
self.status = 'p'
|
||||
del self.data[oid]
|
||||
return not self.data
|
||||
|
||||
@zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
|
||||
class DataRecord(object):
|
||||
"""Abstract base class for iterator protocol"""
|
||||
|
||||
|
||||
version = ''
|
||||
data_txn = None
|
||||
|
||||
def __init__(self, oid, tid, data):
|
||||
self.oid = oid
|
||||
self.tid = tid
|
||||
self.data = data
|
||||
|
||||
def DB(*args, **kw):
|
||||
return ZODB.DB(MappingStorage(), *args, **kw)
|
||||
@@ -0,0 +1,333 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""ZODB-defined exceptions
|
||||
|
||||
$Id$"""
|
||||
|
||||
from ZODB.utils import oid_repr, readable_tid_repr
|
||||
|
||||
# BBB: We moved the two transactions to the transaction package
|
||||
from transaction.interfaces import TransactionError, TransactionFailedError
|
||||
|
||||
import transaction.interfaces
|
||||
|
||||
def _fmt_undo(oid, reason):
|
||||
s = reason and (": %s" % reason) or ""
|
||||
return "Undo error %s%s" % (oid_repr(oid), s)
|
||||
|
||||
def _recon(class_, state):
|
||||
err = class_.__new__(class_)
|
||||
err.__setstate__(state)
|
||||
return err
|
||||
_recon.__no_side_effects__ = True
|
||||
|
||||
class POSError(Exception):
|
||||
"""Persistent object system error."""
|
||||
|
||||
def __reduce__(self):
|
||||
# Copy extra data from internal structures
|
||||
state = self.__dict__.copy()
|
||||
state['args'] = self.args
|
||||
|
||||
return (_recon, (self.__class__, state))
|
||||
|
||||
def __setstate__(self, state):
|
||||
# PyPy doesn't store the 'args' attribute in an instance's
|
||||
# __dict__; instead, it uses what amounts to a slot. Because
|
||||
# we customize the pickled representation to just be a dictionary,
|
||||
# the args would then get lost, leading to unprintable exceptions
|
||||
# and worse. Manually assign to args from the state to be sure
|
||||
# this doesn't happen.
|
||||
super(POSError,self).__setstate__(state)
|
||||
self.args = state['args']
|
||||
|
||||
class POSKeyError(POSError, KeyError):
|
||||
"""Key not found in database."""
|
||||
|
||||
def __str__(self):
|
||||
return oid_repr(self.args[0])
|
||||
|
||||
|
||||
class ConflictError(POSError, transaction.interfaces.TransientError):
|
||||
"""Two transactions tried to modify the same object at once.
|
||||
|
||||
This transaction should be resubmitted.
|
||||
|
||||
Instance attributes:
|
||||
oid : string
|
||||
the OID (8-byte packed string) of the object in conflict
|
||||
class_name : string
|
||||
the fully-qualified name of that object's class
|
||||
message : string
|
||||
a human-readable explanation of the error
|
||||
serials : (string, string)
|
||||
a pair of 8-byte packed strings; these are the serial numbers
|
||||
related to conflict. The first is the revision of object that
|
||||
is in conflict, the currently committed serial. The second is
|
||||
the revision the current transaction read when it started.
|
||||
data : string
|
||||
The database record that failed to commit, used to put the
|
||||
class name in the error message.
|
||||
|
||||
The caller should pass either object or oid as a keyword argument,
|
||||
but not both of them. If object is passed, it should be a
|
||||
persistent object with an _p_oid attribute.
|
||||
"""
|
||||
|
||||
def __init__(self, message=None, object=None, oid=None, serials=None,
|
||||
data=None):
|
||||
if message is None:
|
||||
self.message = "database conflict error"
|
||||
else:
|
||||
self.message = message
|
||||
|
||||
if object is None:
|
||||
self.oid = None
|
||||
self.class_name = None
|
||||
else:
|
||||
self.oid = object._p_oid
|
||||
klass = object.__class__
|
||||
self.class_name = klass.__module__ + "." + klass.__name__
|
||||
|
||||
if oid is not None:
|
||||
assert self.oid is None
|
||||
self.oid = oid
|
||||
|
||||
if data is not None:
|
||||
# avoid circular import chain
|
||||
from ZODB.utils import get_pickle_metadata
|
||||
self.class_name = '.'.join(get_pickle_metadata(data))
|
||||
|
||||
self.serials = serials
|
||||
|
||||
def __str__(self):
|
||||
extras = []
|
||||
if self.oid:
|
||||
extras.append("oid %s" % oid_repr(self.oid))
|
||||
if self.class_name:
|
||||
extras.append("class %s" % self.class_name)
|
||||
if self.serials:
|
||||
current, old = self.serials
|
||||
extras.append("serial this txn started with %s" %
|
||||
readable_tid_repr(old))
|
||||
extras.append("serial currently committed %s" %
|
||||
readable_tid_repr(current))
|
||||
if extras:
|
||||
return "%s (%s)" % (self.message, ", ".join(extras))
|
||||
else:
|
||||
return self.message
|
||||
|
||||
def get_oid(self):
|
||||
return self.oid
|
||||
|
||||
def get_class_name(self):
|
||||
return self.class_name
|
||||
|
||||
def get_old_serial(self):
|
||||
return self.serials[1]
|
||||
|
||||
def get_new_serial(self):
|
||||
return self.serials[0]
|
||||
|
||||
def get_serials(self):
|
||||
return self.serials
|
||||
|
||||
class ReadConflictError(ConflictError):
|
||||
"""Conflict detected when object was loaded.
|
||||
|
||||
An attempt was made to read an object that has changed in another
|
||||
transaction (eg. another thread or process).
|
||||
"""
|
||||
def __init__(self, message=None, object=None, serials=None, **kw):
|
||||
if message is None:
|
||||
message = "database read conflict error"
|
||||
ConflictError.__init__(self, message=message, object=object,
|
||||
serials=serials, **kw)
|
||||
|
||||
class BTreesConflictError(ConflictError):
|
||||
"""A special subclass for BTrees conflict errors."""
|
||||
|
||||
msgs = [# 0; i2 or i3 bucket split; positions are all -1
|
||||
'Conflicting bucket split',
|
||||
|
||||
# 1; keys the same, but i2 and i3 values differ, and both values
|
||||
# differ from i1's value
|
||||
'Conflicting changes',
|
||||
|
||||
# 2; i1's value changed in i2, but key+value deleted in i3
|
||||
'Conflicting delete and change',
|
||||
|
||||
# 3; i1's value changed in i3, but key+value deleted in i2
|
||||
'Conflicting delete and change',
|
||||
|
||||
# 4; i1 and i2 both added the same key, or both deleted the
|
||||
# same key
|
||||
'Conflicting inserts or deletes',
|
||||
|
||||
# 5; i2 and i3 both deleted the same key
|
||||
'Conflicting deletes',
|
||||
|
||||
# 6; i2 and i3 both added the same key
|
||||
'Conflicting inserts',
|
||||
|
||||
# 7; i2 and i3 both deleted the same key, or i2 changed the value
|
||||
# associated with a key and i3 deleted that key
|
||||
'Conflicting deletes, or delete and change',
|
||||
|
||||
# 8; i2 and i3 both deleted the same key, or i3 changed the value
|
||||
# associated with a key and i2 deleted that key
|
||||
'Conflicting deletes, or delete and change',
|
||||
|
||||
# 9; i2 and i3 both deleted the same key
|
||||
'Conflicting deletes',
|
||||
|
||||
# 10; i2 and i3 deleted all the keys, and didn't insert any,
|
||||
# leaving an empty bucket; conflict resolution doesn't have
|
||||
# enough info to unlink an empty bucket from its containing
|
||||
# BTree correctly
|
||||
'Empty bucket from deleting all keys',
|
||||
|
||||
# 11; conflicting changes in an internal BTree node
|
||||
'Conflicting changes in an internal BTree node',
|
||||
|
||||
# 12; i2 or i3 was empty
|
||||
'Empty bucket in a transaction',
|
||||
|
||||
# 13; delete of first key, which causes change to parent node
|
||||
'Delete of first key',
|
||||
]
|
||||
|
||||
def __init__(self, p1, p2, p3, reason):
|
||||
self.p1 = p1
|
||||
self.p2 = p2
|
||||
self.p3 = p3
|
||||
self.reason = reason
|
||||
|
||||
def __repr__(self):
|
||||
return "BTreesConflictError(%d, %d, %d, %d)" % (self.p1,
|
||||
self.p2,
|
||||
self.p3,
|
||||
self.reason)
|
||||
def __str__(self):
|
||||
return "BTrees conflict error at %d/%d/%d: %s" % (
|
||||
self.p1, self.p2, self.p3, self.msgs[self.reason])
|
||||
|
||||
class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
|
||||
"""An object has a persistent reference to a missing object.
|
||||
|
||||
If an object is stored and it has a reference to another object
|
||||
that does not exist (for example, it was deleted by pack), this
|
||||
exception may be raised. Whether a storage supports this feature,
|
||||
it a quality of implementation issue.
|
||||
|
||||
Instance attributes:
|
||||
referer: oid of the object being written
|
||||
missing: referenced oid that does not have a corresponding object
|
||||
"""
|
||||
|
||||
def __init__(self, Aoid, Boid):
|
||||
self.referer = Aoid
|
||||
self.missing = Boid
|
||||
|
||||
def __str__(self):
|
||||
return "from %s to %s" % (oid_repr(self.referer),
|
||||
oid_repr(self.missing))
|
||||
|
||||
|
||||
############################################################################
|
||||
# Only used in storages; versions are no longer supported.
|
||||
|
||||
class VersionError(POSError):
|
||||
"""An error in handling versions occurred."""
|
||||
|
||||
class VersionCommitError(VersionError):
|
||||
"""An invalid combination of versions was used in a version commit."""
|
||||
|
||||
class VersionLockError(VersionError, transaction.interfaces.TransactionError):
|
||||
"""Modification to an object modified in an unsaved version.
|
||||
|
||||
An attempt was made to modify an object that has been modified in an
|
||||
unsaved version.
|
||||
"""
|
||||
############################################################################
|
||||
|
||||
class UndoError(POSError):
|
||||
"""An attempt was made to undo a non-undoable transaction."""
|
||||
|
||||
def __init__(self, reason, oid=None):
|
||||
self._reason = reason
|
||||
self._oid = oid
|
||||
|
||||
def __str__(self):
|
||||
return _fmt_undo(self._oid, self._reason)
|
||||
|
||||
class MultipleUndoErrors(UndoError):
|
||||
"""Several undo errors occurred during a single transaction."""
|
||||
|
||||
def __init__(self, errs):
|
||||
# provide a reason and oid for clients that only look at that
|
||||
UndoError.__init__(self, *errs[0])
|
||||
self._errs = errs
|
||||
|
||||
def __str__(self):
|
||||
return "\n".join([_fmt_undo(*pair) for pair in self._errs])
|
||||
|
||||
class StorageError(POSError):
|
||||
"""Base class for storage based exceptions."""
|
||||
|
||||
class StorageTransactionError(StorageError):
|
||||
"""An operation was invoked for an invalid transaction or state."""
|
||||
|
||||
class StorageSystemError(StorageError):
|
||||
"""Panic! Internal storage error!"""
|
||||
|
||||
class MountedStorageError(StorageError):
|
||||
"""Unable to access mounted storage."""
|
||||
|
||||
class ReadOnlyError(StorageError):
|
||||
"""Unable to modify objects in a read-only storage."""
|
||||
|
||||
class TransactionTooLargeError(StorageTransactionError):
|
||||
"""The transaction exhausted some finite storage resource."""
|
||||
|
||||
class ExportError(POSError):
|
||||
"""An export file doesn't have the right format."""
|
||||
|
||||
class Unsupported(POSError):
|
||||
"""A feature was used that is not supported by the storage."""
|
||||
|
||||
class ReadOnlyHistoryError(POSError):
|
||||
"""Unable to add or modify objects in an historical connection."""
|
||||
|
||||
class InvalidObjectReference(POSError):
|
||||
"""An object contains an invalid reference to another object.
|
||||
|
||||
An invalid reference may be one of:
|
||||
|
||||
o A reference to a wrapped persistent object.
|
||||
|
||||
o A reference to an object in a different database connection.
|
||||
|
||||
TODO: The exception ought to have a member that is the invalid object.
|
||||
"""
|
||||
|
||||
class ConnectionStateError(POSError):
|
||||
"""A Connection isn't in the required state for an operation.
|
||||
|
||||
o An operation such as a load is attempted on a closed connection.
|
||||
|
||||
o An attempt to close a connection is made while the connection is
|
||||
still joined to a transaction (for example, a transaction is in
|
||||
progress, with uncommitted modifications in the connection).
|
||||
"""
|
||||
@@ -0,0 +1,37 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Provide backward compatibility with storages that only have undoLog()."""
|
||||
|
||||
|
||||
class UndoLogCompatible(object):
|
||||
|
||||
def undoInfo(self, first=0, last=-20, specification=None):
|
||||
if specification:
|
||||
# filter(desc) returns true iff `desc` is a "superdict"
|
||||
# of `specification`, meaning that `desc` contains the same
|
||||
# (key, value) pairs as `specification`, and possibly additional
|
||||
# (key, value) pairs. Another way to do this might be
|
||||
# d = desc.copy()
|
||||
# d.update(specification)
|
||||
# return d == desc
|
||||
def filter(desc, spec=specification.items()):
|
||||
get = desc.get
|
||||
for k, v in spec:
|
||||
if get(k, None) != v:
|
||||
return 0
|
||||
return 1
|
||||
else:
|
||||
filter = None
|
||||
|
||||
return self.undoLog(first, last, filter)
|
||||
@@ -0,0 +1,28 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
import sys
|
||||
|
||||
from persistent import TimeStamp
|
||||
from persistent import list
|
||||
from persistent import mapping
|
||||
|
||||
# Backward compat for old imports.
|
||||
sys.modules['ZODB.TimeStamp'] = sys.modules['persistent.TimeStamp']
|
||||
sys.modules['ZODB.PersistentMapping'] = sys.modules['persistent.mapping']
|
||||
sys.modules['ZODB.PersistentList'] = sys.modules['persistent.list']
|
||||
|
||||
del mapping, list, sys
|
||||
|
||||
from ZODB.DB import DB, connection
|
||||
@@ -0,0 +1,160 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2013 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
import sys
|
||||
from six import PY3
|
||||
|
||||
IS_JYTHON = sys.platform.startswith('java')
|
||||
|
||||
_protocol = 3
|
||||
from zodbpickle import binary
|
||||
|
||||
if not PY3:
|
||||
# Python 2.x
|
||||
# PyPy's cPickle doesn't have noload, and noload is broken in Python 2.7,
|
||||
# so we need zodbpickle.
|
||||
# Get the fastest working version we can (PyPy has no fastpickle)
|
||||
try:
|
||||
import zodbpickle.fastpickle as cPickle
|
||||
except ImportError:
|
||||
import zodbpickle.pickle as cPickle
|
||||
Pickler = cPickle.Pickler
|
||||
Unpickler = cPickle.Unpickler
|
||||
dump = cPickle.dump
|
||||
dumps = cPickle.dumps
|
||||
loads = cPickle.loads
|
||||
HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
|
||||
IMPORT_MAPPING = {}
|
||||
NAME_MAPPING = {}
|
||||
FILESTORAGE_MAGIC = b"FS21"
|
||||
else:
|
||||
# Python 3.x: can't use stdlib's pickle because
|
||||
# http://bugs.python.org/issue6784
|
||||
import zodbpickle.pickle
|
||||
HIGHEST_PROTOCOL = 3
|
||||
from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING
|
||||
|
||||
class Pickler(zodbpickle.pickle.Pickler):
|
||||
def __init__(self, f, protocol=None):
|
||||
super(Pickler, self).__init__(f, protocol)
|
||||
|
||||
class Unpickler(zodbpickle.pickle.Unpickler):
|
||||
def __init__(self, f):
|
||||
super(Unpickler, self).__init__(f)
|
||||
|
||||
# Py3: Python 3 doesn't allow assignments to find_global,
|
||||
# instead, find_class can be overridden
|
||||
|
||||
find_global = None
|
||||
|
||||
def find_class(self, modulename, name):
|
||||
if self.find_global is None:
|
||||
return super(Unpickler, self).find_class(modulename, name)
|
||||
return self.find_global(modulename, name)
|
||||
|
||||
def dump(o, f, protocol=None):
|
||||
return zodbpickle.pickle.dump(o, f, protocol)
|
||||
|
||||
def dumps(o, protocol=None):
|
||||
return zodbpickle.pickle.dumps(o, protocol)
|
||||
|
||||
def loads(s):
|
||||
return zodbpickle.pickle.loads(s, encoding='ASCII', errors='bytes')
|
||||
FILESTORAGE_MAGIC = b"FS30"
|
||||
|
||||
|
||||
def PersistentPickler(persistent_id, *args, **kwargs):
|
||||
"""
|
||||
Returns a :class:`Pickler` that will use the given ``persistent_id``
|
||||
to get persistent IDs. The remainder of the arguments are passed to the
|
||||
Pickler itself.
|
||||
|
||||
This covers the differences between Python 2 and 3 and PyPy/zodbpickle.
|
||||
"""
|
||||
p = Pickler(*args, **kwargs)
|
||||
if not PY3:
|
||||
p.inst_persistent_id = persistent_id
|
||||
|
||||
# PyPy uses a python implementation of cPickle/zodbpickle in both Python 2
|
||||
# and Python 3. We can't really detect inst_persistent_id as its
|
||||
# a magic attribute that's not readable, but it doesn't hurt to
|
||||
# simply always assign to persistent_id also
|
||||
p.persistent_id = persistent_id
|
||||
return p
|
||||
|
||||
def PersistentUnpickler(find_global, load_persistent, *args, **kwargs):
|
||||
"""
|
||||
Returns a :class:`Unpickler` that will use the given `find_global` function
|
||||
to locate classes, and the given `load_persistent` function to load
|
||||
objects from a persistent id.
|
||||
|
||||
This covers the differences between Python 2 and 3 and PyPy/zodbpickle.
|
||||
"""
|
||||
unpickler = Unpickler(*args, **kwargs)
|
||||
if find_global is not None:
|
||||
unpickler.find_global = find_global
|
||||
try:
|
||||
unpickler.find_class = find_global # PyPy, zodbpickle, the non-c-accelerated version
|
||||
except AttributeError:
|
||||
pass
|
||||
if load_persistent is not None:
|
||||
unpickler.persistent_load = load_persistent
|
||||
|
||||
return unpickler
|
||||
|
||||
|
||||
try:
|
||||
# XXX: why not just import BytesIO from io?
|
||||
from cStringIO import StringIO as BytesIO
|
||||
except ImportError:
|
||||
# Python 3.x
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
try:
|
||||
# Python 3.x
|
||||
from base64 import decodebytes, encodebytes
|
||||
except ImportError:
|
||||
# Python 2.x
|
||||
from base64 import decodestring as decodebytes, encodestring as encodebytes
|
||||
|
||||
|
||||
# Python 3.x: ``hasattr()`` swallows only AttributeError.
|
||||
def py2_hasattr(obj, name):
|
||||
try:
|
||||
getattr(obj, name)
|
||||
except:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
try:
|
||||
# Py2: simply reexport the builtin
|
||||
long = long
|
||||
except NameError:
|
||||
# Py3
|
||||
long = int
|
||||
INT_TYPES = (int,)
|
||||
else:
|
||||
INT_TYPES = (int, long)
|
||||
|
||||
|
||||
try:
|
||||
TEXT = unicode
|
||||
except NameError: #pragma NO COVER Py3k
|
||||
TEXT = str
|
||||
|
||||
def ascii_bytes(x):
|
||||
if isinstance(x, TEXT):
|
||||
x = x.encode('ascii')
|
||||
return x
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,360 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Broken object support
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import persistent
|
||||
import zope.interface
|
||||
|
||||
import ZODB.interfaces
|
||||
from ZODB._compat import IMPORT_MAPPING
|
||||
from ZODB._compat import NAME_MAPPING
|
||||
|
||||
broken_cache = {}
|
||||
|
||||
@zope.interface.implementer(ZODB.interfaces.IBroken)
|
||||
class Broken(object):
|
||||
"""Broken object base class
|
||||
|
||||
Broken objects are placeholders for objects that can no longer be
|
||||
created because their class has gone away.
|
||||
|
||||
Broken objects don't really do much of anything, except hold their
|
||||
state. The Broken class is used as a base class for creating
|
||||
classes in lieu of missing classes::
|
||||
|
||||
>>> Atall = type('Atall', (Broken, ), {'__module__': 'not.there'})
|
||||
|
||||
The only thing the class can be used for is to create new objects::
|
||||
|
||||
>>> Atall()
|
||||
<broken not.there.Atall instance>
|
||||
>>> Atall().__Broken_newargs__
|
||||
()
|
||||
>>> Atall().__Broken_initargs__
|
||||
()
|
||||
|
||||
>>> Atall(1, 2).__Broken_newargs__
|
||||
(1, 2)
|
||||
>>> Atall(1, 2).__Broken_initargs__
|
||||
(1, 2)
|
||||
|
||||
>>> a = Atall.__new__(Atall, 1, 2)
|
||||
>>> a
|
||||
<broken not.there.Atall instance>
|
||||
>>> a.__Broken_newargs__
|
||||
(1, 2)
|
||||
>>> a.__Broken_initargs__
|
||||
|
||||
You can't modify broken objects::
|
||||
|
||||
>>> a.x = 1
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BrokenModified: Can't change broken objects
|
||||
|
||||
But you can set their state::
|
||||
|
||||
>>> a.__setstate__({'x': 1, })
|
||||
|
||||
You can pickle broken objects::
|
||||
|
||||
>>> r = a.__reduce__()
|
||||
>>> len(r)
|
||||
3
|
||||
>>> r[0] is rebuild
|
||||
True
|
||||
>>> r[1]
|
||||
('not.there', 'Atall', 1, 2)
|
||||
>>> r[2]
|
||||
{'x': 1}
|
||||
|
||||
>>> from ZODB._compat import dumps
|
||||
>>> from ZODB._compat import loads
|
||||
>>> from ZODB._compat import _protocol
|
||||
>>> a2 = loads(dumps(a, _protocol))
|
||||
>>> a2
|
||||
<broken not.there.Atall instance>
|
||||
>>> a2.__Broken_newargs__
|
||||
(1, 2)
|
||||
>>> a2.__Broken_initargs__
|
||||
>>> a2.__Broken_state__
|
||||
{'x': 1}
|
||||
|
||||
Cleanup::
|
||||
|
||||
>>> broken_cache.clear()
|
||||
"""
|
||||
|
||||
|
||||
__Broken_state__ = __Broken_initargs__ = None
|
||||
|
||||
__name__ = 'broken object'
|
||||
|
||||
def __new__(class_, *args):
|
||||
result = object.__new__(class_)
|
||||
result.__dict__['__Broken_newargs__'] = args
|
||||
return result
|
||||
|
||||
def __init__(self, *args):
|
||||
self.__dict__['__Broken_initargs__'] = args
|
||||
|
||||
def __reduce__(self):
|
||||
"""We pickle broken objects in hope of being able to fix them later
|
||||
"""
|
||||
return (rebuild,
|
||||
((self.__class__.__module__, self.__class__.__name__)
|
||||
+ self.__Broken_newargs__),
|
||||
self.__Broken_state__,
|
||||
)
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__dict__['__Broken_state__'] = state
|
||||
|
||||
def __repr__(self):
|
||||
return "<broken %s.%s instance>" % (
|
||||
self.__class__.__module__, self.__class__.__name__)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
raise BrokenModified("Can't change broken objects")
|
||||
|
||||
def find_global(modulename, globalname,
|
||||
# These are *not* optimizations. Callers can override these.
|
||||
Broken=Broken, type=type,
|
||||
):
|
||||
"""Find a global object, returning a broken class if it can't be found.
|
||||
|
||||
This function looks up global variable in modules::
|
||||
|
||||
>>> import sys
|
||||
>>> find_global('sys', 'path') is sys.path
|
||||
True
|
||||
|
||||
>>> find_global('__builtin__', 'object') is object
|
||||
True
|
||||
|
||||
If an object can't be found, a broken class is returned::
|
||||
|
||||
>>> broken = find_global('ZODB.not.there', 'atall')
|
||||
>>> issubclass(broken, Broken)
|
||||
True
|
||||
>>> broken.__module__
|
||||
'ZODB.not.there'
|
||||
>>> broken.__name__
|
||||
'atall'
|
||||
|
||||
Broken classes are cached::
|
||||
|
||||
>>> find_global('ZODB.not.there', 'atall') is broken
|
||||
True
|
||||
|
||||
If we "repair" a missing global::
|
||||
|
||||
>>> class ZODBnotthere(object):
|
||||
... atall = []
|
||||
|
||||
>>> sys.modules['ZODB.not'] = ZODBnotthere
|
||||
>>> sys.modules['ZODB.not.there'] = ZODBnotthere
|
||||
|
||||
we can then get the repaired value::
|
||||
|
||||
>>> find_global('ZODB.not.there', 'atall') is ZODBnotthere.atall
|
||||
True
|
||||
|
||||
Of course, if we break it again::
|
||||
|
||||
>>> del sys.modules['ZODB.not']
|
||||
>>> del sys.modules['ZODB.not.there']
|
||||
|
||||
we get the broken value::
|
||||
|
||||
>>> find_global('ZODB.not.there', 'atall') is broken
|
||||
True
|
||||
|
||||
Cleanup::
|
||||
|
||||
>>> broken_cache.clear()
|
||||
"""
|
||||
|
||||
if (modulename, globalname) in NAME_MAPPING:
|
||||
modulename, globalname = NAME_MAPPING[(modulename, globalname)]
|
||||
if modulename in IMPORT_MAPPING:
|
||||
modulename = IMPORT_MAPPING[modulename]
|
||||
|
||||
# short circuit common case:
|
||||
try:
|
||||
return getattr(sys.modules[modulename], globalname)
|
||||
except (AttributeError, KeyError):
|
||||
pass
|
||||
|
||||
try:
|
||||
__import__(modulename)
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
module = sys.modules[modulename]
|
||||
try:
|
||||
return getattr(module, globalname)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
return broken_cache[(modulename, globalname)]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
class_ = type(globalname, (Broken, ), {'__module__': modulename})
|
||||
broken_cache[(modulename, globalname)] = class_
|
||||
return class_
|
||||
|
||||
def rebuild(modulename, globalname, *args):
|
||||
"""Recreate a broken object, possibly recreating the missing class
|
||||
|
||||
This functions unpickles broken objects::
|
||||
|
||||
>>> broken = rebuild('ZODB.notthere', 'atall', 1, 2)
|
||||
>>> broken
|
||||
<broken ZODB.notthere.atall instance>
|
||||
>>> broken.__Broken_newargs__
|
||||
(1, 2)
|
||||
|
||||
If we "repair" the brokenness::
|
||||
|
||||
>>> class notthere(object): # fake notthere module
|
||||
... class atall(object):
|
||||
... def __new__(self, *args):
|
||||
... ob = object.__new__(self)
|
||||
... ob.args = args
|
||||
... return ob
|
||||
... def __repr__(self):
|
||||
... return 'atall %s %s' % self.args
|
||||
|
||||
>>> sys.modules['ZODB.notthere'] = notthere
|
||||
|
||||
>>> rebuild('ZODB.notthere', 'atall', 1, 2)
|
||||
atall 1 2
|
||||
|
||||
>>> del sys.modules['ZODB.notthere']
|
||||
|
||||
Cleanup::
|
||||
|
||||
>>> broken_cache.clear()
|
||||
|
||||
"""
|
||||
class_ = find_global(modulename, globalname)
|
||||
return class_.__new__(class_, *args)
|
||||
|
||||
class BrokenModified(TypeError):
|
||||
"""Attempt to modify a broken object
|
||||
"""
|
||||
|
||||
class PersistentBroken(Broken, persistent.Persistent):
|
||||
r"""Persistent broken objects
|
||||
|
||||
Persistent broken objects are used for broken objects that are
|
||||
also persistent. In addition to having to track the original
|
||||
object data, they need to handle persistent meta data.
|
||||
|
||||
Persistent broken classes are created from existing broken classes
|
||||
using the persistentBroken, function::
|
||||
|
||||
>>> Atall = type('Atall', (Broken, ), {'__module__': 'not.there'})
|
||||
>>> PAtall = persistentBroken(Atall)
|
||||
|
||||
(Note that we always get the *same* persistent broken class
|
||||
for a given broken class::
|
||||
|
||||
>>> persistentBroken(Atall) is PAtall
|
||||
True
|
||||
|
||||
)
|
||||
|
||||
Persistent broken classes work a lot like broken classes::
|
||||
|
||||
>>> a = PAtall.__new__(PAtall, 1, 2)
|
||||
>>> a
|
||||
<persistent broken not.there.Atall instance None>
|
||||
>>> a.__Broken_newargs__
|
||||
(1, 2)
|
||||
>>> a.__Broken_initargs__
|
||||
>>> a.x = 1
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BrokenModified: Can't change broken objects
|
||||
|
||||
Unlike regular broken objects, persistent broken objects keep
|
||||
track of persistence meta data:
|
||||
|
||||
>>> a._p_oid = '\0\0\0\0****'
|
||||
>>> a
|
||||
<persistent broken not.there.Atall instance '\x00\x00\x00\x00****'>
|
||||
|
||||
and persistent broken objects aren't directly picklable:
|
||||
|
||||
>>> a.__reduce__() # doctest: +NORMALIZE_WHITESPACE
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BrokenModified:
|
||||
<persistent broken not.there.Atall instance '\x00\x00\x00\x00****'>
|
||||
|
||||
but you can get their state:
|
||||
|
||||
>>> a.__setstate__({'y': 2})
|
||||
>>> a.__getstate__()
|
||||
{'y': 2}
|
||||
|
||||
Cleanup::
|
||||
|
||||
>>> broken_cache.clear()
|
||||
|
||||
"""
|
||||
|
||||
def __new__(class_, *args):
|
||||
result = persistent.Persistent.__new__(class_)
|
||||
result.__dict__['__Broken_newargs__'] = args
|
||||
return result
|
||||
|
||||
def __reduce__(self, *args):
|
||||
raise BrokenModified(self)
|
||||
|
||||
def __getstate__(self):
|
||||
return self.__Broken_state__
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
if name.startswith('_p_'):
|
||||
persistent.Persistent.__setattr__(self, name, value)
|
||||
else:
|
||||
raise BrokenModified("Can't change broken objects")
|
||||
|
||||
def __repr__(self):
|
||||
return "<persistent broken %s.%s instance %r>" % (
|
||||
self.__class__.__module__, self.__class__.__name__,
|
||||
self._p_oid)
|
||||
|
||||
def __getnewargs__(self):
|
||||
return self.__Broken_newargs__
|
||||
|
||||
def persistentBroken(class_):
|
||||
try:
|
||||
return class_.__dict__['__Broken_Persistent__']
|
||||
except KeyError:
|
||||
class_.__Broken_Persistent__ = (
|
||||
type(class_.__name__,
|
||||
(PersistentBroken, class_),
|
||||
{'__module__': class_.__module__},
|
||||
)
|
||||
)
|
||||
return class_.__dict__['__Broken_Persistent__']
|
||||
@@ -0,0 +1,199 @@
|
||||
=======================
|
||||
Collabortation Diagrams
|
||||
=======================
|
||||
|
||||
This file contains several collaboration diagrams for the ZODB.
|
||||
|
||||
Simple fetch, modify, commit
|
||||
============================
|
||||
|
||||
Participants
|
||||
------------
|
||||
|
||||
- ``DB``: ``ZODB.DB.DB``
|
||||
- ``C``: ``ZODB.Connection.Connection``
|
||||
- ``S``: ``ZODB.FileStorage.FileStorage``
|
||||
- ``T``: ``transaction.interfaces.ITransaction``
|
||||
- ``TM``: ``transaction.interfaces.ITransactionManager``
|
||||
- ``o1``, ``o2``, ...: pre-existing persistent objects
|
||||
|
||||
Scenario
|
||||
--------
|
||||
|
||||
::
|
||||
|
||||
DB.open()
|
||||
create C
|
||||
TM.registerSynch(C)
|
||||
TM.begin()
|
||||
create T
|
||||
C.get(1) # fetches o1
|
||||
C.get(2) # fetches o2
|
||||
C.get(3) # fetches o3
|
||||
o1.modify() # anything that modifies o1
|
||||
C.register(o1)
|
||||
T.join(C)
|
||||
o2.modify()
|
||||
C.register(o2)
|
||||
# T.join(C) does not happen again
|
||||
o1.modify()
|
||||
# C.register(o1) doesn't happen again, because o1 was already
|
||||
# in the changed state.
|
||||
T.commit()
|
||||
C.beforeCompletion(T)
|
||||
C.tpc_begin(T)
|
||||
S.tpc_begin(T)
|
||||
C.commit(T)
|
||||
S.store(1, ..., T)
|
||||
S.store(2, ..., T)
|
||||
# o3 is not stored, because it wasn't modified
|
||||
C.tpc_vote(T)
|
||||
S.tpc_vote(T)
|
||||
C.tpc_finish(T)
|
||||
S.tpc_finish(T, f) # f is a callback function, which arranges
|
||||
# to call DB.invalidate (next)
|
||||
DB.invalidate(tid, {1: 1, 2: 1}, C)
|
||||
C2.invalidate(tid, {1: 1, 2: 1}) # for all connections
|
||||
# C2 to DB, where C2
|
||||
# is not C
|
||||
TM.free(T)
|
||||
C.afterCompletion(T)
|
||||
C._flush_invalidations()
|
||||
# Processes invalidations that may have come in from other
|
||||
# transactions.
|
||||
|
||||
|
||||
Simple fetch, modify, abort
|
||||
===========================
|
||||
|
||||
Participants
|
||||
------------
|
||||
|
||||
- ``DB``: ``ZODB.DB.DB``
|
||||
- ``C``: ``ZODB.Connection.Connection``
|
||||
- ``S``: ``ZODB.FileStorage.FileStorage``
|
||||
- ``T``: ``transaction.interfaces.ITransaction``
|
||||
- ``TM``: ``transaction.interfaces.ITransactionManager``
|
||||
- ``o1``, ``o2``, ...: pre-existing persistent objects
|
||||
|
||||
Scenario
|
||||
--------
|
||||
|
||||
::
|
||||
|
||||
DB.open()
|
||||
create C
|
||||
TM.registerSynch(C)
|
||||
TM.begin()
|
||||
create T
|
||||
C.get(1) # fetches o1
|
||||
C.get(2) # fetches o2
|
||||
C.get(3) # fetches o3
|
||||
o1.modify() # anything that modifies o1
|
||||
C.register(o1)
|
||||
T.join(C)
|
||||
o2.modify()
|
||||
C.register(o2)
|
||||
# T.join(C) does not happen again
|
||||
o1.modify()
|
||||
# C.register(o1) doesn't happen again, because o1 was already
|
||||
# in the changed state.
|
||||
T.abort()
|
||||
C.beforeCompletion(T)
|
||||
C.abort(T)
|
||||
C._cache.invalidate(1) # toss changes to o1
|
||||
C._cache.invalidate(2) # toss changes to o2
|
||||
# o3 wasn't modified, and its cache entry isn't invalidated.
|
||||
TM.free(T)
|
||||
C.afterCompletion(T)
|
||||
C._flush_invalidations()
|
||||
# Processes invalidations that may have come in from other
|
||||
# transactions.
|
||||
|
||||
|
||||
Rollback of a savepoint
|
||||
=======================
|
||||
|
||||
Participants
|
||||
------------
|
||||
|
||||
- ``T``: ``transaction.interfaces.ITransaction``
|
||||
- ``o1``, ``o2``, ``o3``: some persistent objects
|
||||
- ``C1``, ``C2``, ``C3``: resource managers
|
||||
- ``S1``, ``S2``: Transaction savepoint objects
|
||||
- ``s11``, ``s21``, ``s22``: resource-manager savepoints
|
||||
|
||||
Scenario
|
||||
--------
|
||||
|
||||
::
|
||||
|
||||
create T
|
||||
o1.modify()
|
||||
C1.regisiter(o1)
|
||||
T.join(C1)
|
||||
T.savepoint()
|
||||
C1.savepoint()
|
||||
return s11
|
||||
return S1 = Savepoint(T, [r11])
|
||||
o1.modify()
|
||||
C1.regisiter(o1)
|
||||
o2.modify()
|
||||
C2.regisiter(o2)
|
||||
T.join(C2)
|
||||
T.savepoint()
|
||||
C1.savepoint()
|
||||
return s21
|
||||
C2.savepoint()
|
||||
return s22
|
||||
return S2 = Savepoint(T, [r21, r22])
|
||||
o3.modify()
|
||||
C3.regisiter(o3)
|
||||
T.join(C3)
|
||||
S1.rollback()
|
||||
S2.rollback()
|
||||
T.discard()
|
||||
C1.discard()
|
||||
C2.discard()
|
||||
C3.discard()
|
||||
o3.invalidate()
|
||||
S2.discard()
|
||||
s21.discard() # roll back changes since previous, which is r11
|
||||
C1.discard(s21)
|
||||
o1.invalidate()
|
||||
# truncates temporary storage to s21's position
|
||||
s22.discard() # roll back changes since previous, which is r11
|
||||
C1.discard(s22)
|
||||
o2.invalidate()
|
||||
# truncates temporary storage to beginning, because
|
||||
# s22 was the first savepoint. (Perhaps conection
|
||||
# savepoints record the log position before the
|
||||
# data were written, which is 0 in this case.
|
||||
T.commit()
|
||||
C1.beforeCompletion(T)
|
||||
C2.beforeCompletion(T)
|
||||
C3.beforeCompletion(T)
|
||||
C1.tpc_begin(T)
|
||||
S1.tpc_begin(T)
|
||||
C2.tpc_begin(T)
|
||||
C3.tpc_begin(T)
|
||||
C1.commit(T)
|
||||
S1.store(1, ..., T)
|
||||
C2.commit(T)
|
||||
C3.commit(T)
|
||||
C1.tpc_vote(T)
|
||||
S1.tpc_vote(T)
|
||||
C2.tpc_vote(T)
|
||||
C3.tpc_vote(T)
|
||||
C1.tpc_finish(T)
|
||||
S1.tpc_finish(T, f) # f is a callback function, which arranges
|
||||
c# to call DB.invalidate (next)
|
||||
DB.invalidate(tid, {1: 1}, C)
|
||||
TM.free(T)
|
||||
C1.afterCompletion(T)
|
||||
C1._flush_invalidations()
|
||||
C2.afterCompletion(T)
|
||||
C2._flush_invalidations()
|
||||
C3.afterCompletion(T)
|
||||
C3._flush_invalidations()
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
<component prefix="ZODB.config">
|
||||
|
||||
<!-- TODO needs descriptions for everything -->
|
||||
|
||||
<abstracttype name="ZODB.storage"/>
|
||||
<abstracttype name="ZODB.database"/>
|
||||
|
||||
<sectiontype name="filestorage" datatype=".FileStorage"
|
||||
implements="ZODB.storage">
|
||||
<key name="path" required="yes" datatype="existing-dirpath">
|
||||
<description>
|
||||
Path name to the main storage file. The names for
|
||||
supplemental files, including index and lock files, will be
|
||||
computed from this.
|
||||
</description>
|
||||
</key>
|
||||
<key name="blob-dir" required="no" datatype="existing-dirpath">
|
||||
<description>
|
||||
If supplied, the file storage will provide blob support and this
|
||||
is the name of a directory to hold blob data. The directory will
|
||||
be created if it doesn't exist. If no value (or an empty value)
|
||||
is provided, then no blob support will be provided. (You can still
|
||||
use a BlobStorage to provide blob support.)
|
||||
</description>
|
||||
</key>
|
||||
<key name="create" datatype="boolean">
|
||||
<description>
|
||||
Flag that indicates whether the storage should be truncated if
|
||||
it already exists.
|
||||
</description>
|
||||
</key>
|
||||
<key name="read-only" datatype="boolean">
|
||||
<description>
|
||||
If true, only reads may be executed against the storage. Note
|
||||
that the "pack" operation is not considered a write operation
|
||||
and is still allowed on a read-only filestorage.
|
||||
</description>
|
||||
</key>
|
||||
<key name="quota" datatype="byte-size">
|
||||
<description>
|
||||
Maximum allowed size of the storage file. Operations which
|
||||
would cause the size of the storage to exceed the quota will
|
||||
result in a ZODB.FileStorage.FileStorageQuotaError being
|
||||
raised.
|
||||
</description>
|
||||
</key>
|
||||
<key name="packer" datatype="string">
|
||||
<description>
|
||||
The dotted name (dotted module name and object name) of a
|
||||
packer object. This is used to provide an alternative pack
|
||||
implementation.
|
||||
</description>
|
||||
</key>
|
||||
<key name="pack-gc" datatype="boolean" default="true">
|
||||
<description>
|
||||
If false, then no garbage collection will be performed when
|
||||
packing. This can make packing go much faster and can avoid
|
||||
problems when objects are referenced only from other
|
||||
databases.
|
||||
</description>
|
||||
</key>
|
||||
<key name="pack-keep-old" datatype="boolean" default="true">
|
||||
<description>
|
||||
If true, a copy of the database before packing is kept in a
|
||||
".old" file.
|
||||
</description>
|
||||
</key>
|
||||
</sectiontype>
|
||||
|
||||
<sectiontype name="mappingstorage" datatype=".MappingStorage"
|
||||
implements="ZODB.storage">
|
||||
<key name="name" default="Mapping Storage">
|
||||
<description>
|
||||
The storage name, used by the
|
||||
:meth:`~ZODB.interfaces.IStorage.getName` and
|
||||
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
|
||||
</description>
|
||||
</key>
|
||||
</sectiontype>
|
||||
|
||||
<!-- The BDB storages probably need to be revised somewhat still.
|
||||
The extension relationship seems a little odd.
|
||||
-->
|
||||
<sectiontype name="fullstorage" datatype=".BDBFullStorage"
|
||||
implements="ZODB.storage">
|
||||
<key name="envdir" required="yes" />
|
||||
<key name="interval" datatype="time-interval" default="2m" />
|
||||
<key name="kbyte" datatype="integer" default="0" />
|
||||
<key name="min" datatype="integer" default="0" />
|
||||
<key name="logdir" />
|
||||
<key name="cachesize" datatype="byte-size" default="128MB" />
|
||||
<key name="frequency" datatype="time-interval" default="0" />
|
||||
<key name="packtime" datatype="time-interval" default="4h" />
|
||||
<key name="gcpack" datatype="integer" default="0" />
|
||||
<key name="read-only" datatype="boolean" default="off"/>
|
||||
</sectiontype>
|
||||
|
||||
<sectiontype name="minimalstorage" datatype=".BDBMinimalStorage"
|
||||
implements="ZODB.storage" extends="fullstorage"/>
|
||||
|
||||
<sectiontype name="zeoclient" datatype=".ZEOClient"
|
||||
implements="ZODB.storage">
|
||||
<multikey name="server" datatype="socket-connection-address" required="yes"/>
|
||||
<key name="blob-dir" required="no">
|
||||
<description>
|
||||
Path name to the blob cache directory.
|
||||
</description>
|
||||
</key>
|
||||
<key name="shared-blob-dir" required="no" default="no"
|
||||
datatype="boolean">
|
||||
<description>
|
||||
Tells whether the cache is a shared writable directory
|
||||
and that the ZEO protocol should not transfer the file
|
||||
but only the filename when committing.
|
||||
</description>
|
||||
</key>
|
||||
<key name="blob-cache-size" required="no" datatype="byte-size">
|
||||
<description>
|
||||
Maximum size of the ZEO blob cache, in bytes. If not set, then
|
||||
the cache size isn't checked and the blob directory will
|
||||
grow without bound.
|
||||
|
||||
This option is ignored if shared_blob_dir is true.
|
||||
</description>
|
||||
</key>
|
||||
<key name="blob-cache-size-check" required="no" datatype="integer">
|
||||
<description>
|
||||
ZEO check size as percent of blob_cache_size. The ZEO
|
||||
cache size will be checked when this many bytes have been
|
||||
loaded into the cache. Defaults to 10% of the blob cache
|
||||
size. This option is ignored if shared_blob_dir is true.
|
||||
</description>
|
||||
</key>
|
||||
<key name="storage" default="1">
|
||||
<description>
|
||||
The name of the storage that the client wants to use. If the
|
||||
ZEO server serves more than one storage, the client selects
|
||||
the storage it wants to use by name. The default name is '1',
|
||||
which is also the default name for the ZEO server.
|
||||
</description>
|
||||
</key>
|
||||
<key name="cache-size" datatype="byte-size" default="20MB">
|
||||
<description>
|
||||
The maximum size of the client cache, in bytes, KB or MB.
|
||||
</description>
|
||||
</key>
|
||||
<key name="name" default="">
|
||||
<description>
|
||||
The storage name. If unspecified, the address of the server
|
||||
will be used as the name.
|
||||
</description>
|
||||
</key>
|
||||
<key name="client">
|
||||
<description>
|
||||
Enables persistent cache files. The string passed here is
|
||||
used to construct the cache filenames. If it is not
|
||||
specified, the client creates a temporary cache that will
|
||||
only be used by the current object.
|
||||
</description>
|
||||
</key>
|
||||
<key name="var">
|
||||
<description>
|
||||
The directory where persistent cache files are stored. By
|
||||
default cache files, if they are persistent, are stored in
|
||||
the current directory.
|
||||
</description>
|
||||
</key>
|
||||
<key name="min-disconnect-poll" datatype="integer" default="5">
|
||||
<description>
|
||||
The minimum delay in seconds between attempts to connect to
|
||||
the server, in seconds. Defaults to 5 seconds.
|
||||
</description>
|
||||
</key>
|
||||
<key name="max-disconnect-poll" datatype="integer" default="300">
|
||||
<description>
|
||||
The maximum delay in seconds between attempts to connect to
|
||||
the server, in seconds. Defaults to 300 seconds.
|
||||
</description>
|
||||
</key>
|
||||
<key name="wait" datatype="boolean" default="on">
|
||||
<description>
|
||||
A boolean indicating whether the constructor should wait
|
||||
for the client to connect to the server and verify the cache
|
||||
before returning. The default is true.
|
||||
</description>
|
||||
</key>
|
||||
<key name="read-only" datatype="boolean" default="off">
|
||||
<description>
|
||||
A flag indicating whether this should be a read-only storage,
|
||||
defaulting to false (i.e. writing is allowed by default).
|
||||
</description>
|
||||
</key>
|
||||
<key name="read-only-fallback" datatype="boolean" default="off">
|
||||
<description>
|
||||
A flag indicating whether a read-only remote storage should be
|
||||
acceptable as a fall-back when no writable storages are
|
||||
available. Defaults to false. At most one of read_only and
|
||||
read_only_fallback should be true.
|
||||
</description>
|
||||
</key>
|
||||
<key name="username" required="no">
|
||||
<description>
|
||||
The authentication user name of the server.
|
||||
</description>
|
||||
</key>
|
||||
<key name="password" required="no">
|
||||
<description>
|
||||
The authentication password of the server.
|
||||
</description>
|
||||
</key>
|
||||
<key name="realm" required="no">
|
||||
<description>
|
||||
The authentication realm of the server. Some authentication
|
||||
schemes use a realm to identify the logic set of user names
|
||||
that are accepted by this server.
|
||||
</description>
|
||||
</key>
|
||||
<!-- DM 2008-05-15: added -->
|
||||
<key name="drop-cache-rather-verify" datatype="boolean" default="off">
|
||||
<description>
|
||||
A flag indicating whether the client cache should be dropped
|
||||
instead of an expensive verification.
|
||||
</description>
|
||||
</key>
|
||||
<key name="client-label" required="no">
|
||||
<description>
|
||||
A label for the client in server logs
|
||||
</description>
|
||||
</key>
|
||||
</sectiontype>
|
||||
|
||||
<sectiontype name="demostorage" datatype=".DemoStorage"
|
||||
implements="ZODB.storage">
|
||||
<key name="name">
|
||||
<description>
|
||||
The storage name, used by the
|
||||
:meth:`~ZODB.interfaces.IStorage.getName` and
|
||||
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
|
||||
</description>
|
||||
</key>
|
||||
<multisection type="ZODB.storage" name="*" attribute="factories" />
|
||||
</sectiontype>
|
||||
|
||||
|
||||
<sectiontype name="zodb" datatype=".ZODBDatabase"
|
||||
implements="ZODB.database">
|
||||
<section type="ZODB.storage" name="*" attribute="storage"/>
|
||||
<key name="cache-size" datatype="integer" default="5000">
|
||||
<description>
|
||||
Target size, in number of objects, of each connection's
|
||||
object cache.
|
||||
</description>
|
||||
</key>
|
||||
<key name="cache-size-bytes" datatype="byte-size" default="0">
|
||||
<description>
|
||||
Target size, in total estimated size for objects, of each connection's
|
||||
object cache.
|
||||
"0" means no limit.
|
||||
</description>
|
||||
</key>
|
||||
<key name="large-record-size" datatype="byte-size" default="16MB">
|
||||
<description>
|
||||
When object records are saved
|
||||
that are larger than this, a warning is issued,
|
||||
suggesting that blobs should be used instead.
|
||||
</description>
|
||||
</key>
|
||||
<key name="pool-size" datatype="integer" default="7">
|
||||
<description>
|
||||
The expected maximum number of simultaneously open connections.
|
||||
There is no hard limit (as many connections as are requested
|
||||
will be opened, until system resources are exhausted). Exceeding
|
||||
pool-size connections causes a warning message to be logged,
|
||||
and exceeding twice pool-size connections causes a critical
|
||||
message to be logged.
|
||||
</description>
|
||||
</key>
|
||||
<key name="pool-timeout" datatype="time-interval">
|
||||
<description>
|
||||
The minimum interval that an unused (non-historical)
|
||||
connection should be kept.
|
||||
</description>
|
||||
</key>
|
||||
<key name="historical-pool-size" datatype="integer" default="3">
|
||||
<description>
|
||||
The expected maximum total number of historical connections
|
||||
simultaneously open.
|
||||
</description>
|
||||
</key>
|
||||
<key name="historical-cache-size" datatype="integer" default="1000">
|
||||
<description>
|
||||
Target size, in number of objects, of each historical connection's
|
||||
object cache.
|
||||
</description>
|
||||
</key>
|
||||
<key name="historical-cache-size-bytes" datatype="byte-size" default="0">
|
||||
<description>
|
||||
Target size, in total estimated size of objects, of each historical connection's
|
||||
object cache.
|
||||
</description>
|
||||
</key>
|
||||
<key name="historical-timeout" datatype="time-interval"
|
||||
default="5m">
|
||||
<description>
|
||||
The minimum interval that an unused historical connection should be
|
||||
kept.
|
||||
</description>
|
||||
</key>
|
||||
<key name="database-name">
|
||||
<description>
|
||||
When multi-databases are in use, this is the name given to this
|
||||
database in the collection. The name must be unique across all
|
||||
databases in the collection. The collection must also be given
|
||||
a mapping from its databases' names to their databases, but that
|
||||
cannot be specified in a ZODB config file. Applications using
|
||||
multi-databases typical supply a way to configure the mapping in
|
||||
their own config files, using the "databases" parameter of a DB
|
||||
constructor.
|
||||
</description>
|
||||
</key>
|
||||
<key name="allow-implicit-cross-references" datatype="boolean">
|
||||
<description>
|
||||
If set to false, implicit cross references (the only kind
|
||||
currently possible) are disallowed.
|
||||
</description>
|
||||
</key>
|
||||
|
||||
</sectiontype>
|
||||
|
||||
<sectiontype name="blobstorage" datatype=".BlobStorage"
|
||||
implements="ZODB.storage">
|
||||
<key name="blob-dir" required="yes">
|
||||
<description>
|
||||
Path name to the blob storage directory.
|
||||
</description>
|
||||
</key>
|
||||
<section type="ZODB.storage" name="*" attribute="base"/>
|
||||
</sectiontype>
|
||||
|
||||
|
||||
|
||||
</component>
|
||||
@@ -0,0 +1,274 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Open database and storage from a configuration."""
|
||||
import os
|
||||
import ZConfig
|
||||
import ZODB
|
||||
|
||||
try:
|
||||
from cStringIO import StringIO
|
||||
except ImportError:
|
||||
# Py3
|
||||
from io import StringIO
|
||||
|
||||
|
||||
db_schema_path = os.path.join(ZODB.__path__[0], "config.xml")
|
||||
_db_schema = None
|
||||
|
||||
s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml")
|
||||
_s_schema = None
|
||||
|
||||
def getDbSchema():
|
||||
global _db_schema
|
||||
if _db_schema is None:
|
||||
_db_schema = ZConfig.loadSchema(db_schema_path)
|
||||
return _db_schema
|
||||
|
||||
def getStorageSchema():
|
||||
global _s_schema
|
||||
if _s_schema is None:
|
||||
_s_schema = ZConfig.loadSchema(s_schema_path)
|
||||
return _s_schema
|
||||
|
||||
def databaseFromString(s):
|
||||
"""Create a database from a database-configuration string.
|
||||
|
||||
The string must contain one or more :ref:`zodb
|
||||
<database-text-configuration>` sections.
|
||||
|
||||
The database defined by the first section is returned.
|
||||
|
||||
If :ref:`more than one zodb section is provided
|
||||
<multidatabase-text-configuration>`, a multi-database
|
||||
configuration will be created and all of the databases will be
|
||||
available in the returned database's ``databases`` attribute.
|
||||
"""
|
||||
return databaseFromFile(StringIO(s))
|
||||
|
||||
def databaseFromFile(f):
|
||||
"""Create a database from a file object that provides configuration.
|
||||
|
||||
See :func:`databaseFromString`.
|
||||
"""
|
||||
config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
|
||||
return databaseFromConfig(config.database)
|
||||
|
||||
def databaseFromURL(url):
|
||||
"""Load a database from URL (or file name) that provides configuration.
|
||||
|
||||
See :func:`databaseFromString`.
|
||||
"""
|
||||
config, handler = ZConfig.loadConfig(getDbSchema(), url)
|
||||
return databaseFromConfig(config.database)
|
||||
|
||||
def databaseFromConfig(database_factories):
|
||||
databases = {}
|
||||
first = None
|
||||
for factory in database_factories:
|
||||
db = factory.open(databases)
|
||||
if first is None:
|
||||
first = db
|
||||
|
||||
return first
|
||||
|
||||
def storageFromString(s):
|
||||
"""Create a storage from a storage-configuration string.
|
||||
"""
|
||||
return storageFromFile(StringIO(s))
|
||||
|
||||
def storageFromFile(f):
|
||||
"""Create a storage from a file object providing storage-configuration.
|
||||
"""
|
||||
config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
|
||||
return storageFromConfig(config.storage)
|
||||
|
||||
def storageFromURL(url):
|
||||
"""\
|
||||
Create a storage from a URL (or file name) providing storage-configuration.
|
||||
"""
|
||||
config, handler = ZConfig.loadConfig(getStorageSchema(), url)
|
||||
return storageFromConfig(config.storage)
|
||||
|
||||
def storageFromConfig(section):
|
||||
return section.open()
|
||||
|
||||
class BaseConfig(object):
|
||||
"""Object representing a configured storage or database.
|
||||
|
||||
Methods:
|
||||
|
||||
open() -- open and return the configured object
|
||||
|
||||
Attributes:
|
||||
|
||||
name -- name of the storage
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.name = config.getSectionName()
|
||||
|
||||
def open(self, database_name='unnamed', databases=None):
|
||||
"""Open and return the storage object."""
|
||||
raise NotImplementedError
|
||||
|
||||
class ZODBDatabase(BaseConfig):
|
||||
|
||||
def open(self, databases=None):
|
||||
section = self.config
|
||||
storage = section.storage.open()
|
||||
options = {}
|
||||
|
||||
def _option(name, oname=None):
|
||||
v = getattr(section, name)
|
||||
if v is not None:
|
||||
if oname is None:
|
||||
oname = name
|
||||
options[oname] = v
|
||||
|
||||
_option('pool_timeout')
|
||||
_option('allow_implicit_cross_references', 'xrefs')
|
||||
_option('large_record_size')
|
||||
|
||||
try:
|
||||
return ZODB.DB(
|
||||
storage,
|
||||
pool_size=section.pool_size,
|
||||
cache_size=section.cache_size,
|
||||
cache_size_bytes=section.cache_size_bytes,
|
||||
historical_pool_size=section.historical_pool_size,
|
||||
historical_cache_size=section.historical_cache_size,
|
||||
historical_cache_size_bytes=section.historical_cache_size_bytes,
|
||||
historical_timeout=section.historical_timeout,
|
||||
database_name=section.database_name or self.name or '',
|
||||
databases=databases,
|
||||
**options)
|
||||
except:
|
||||
storage.close()
|
||||
raise
|
||||
|
||||
class MappingStorage(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
from ZODB.MappingStorage import MappingStorage
|
||||
return MappingStorage(self.config.name)
|
||||
|
||||
class DemoStorage(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
base = changes = None
|
||||
for factory in self.config.factories:
|
||||
if factory.name == 'changes':
|
||||
changes = factory.open()
|
||||
else:
|
||||
if base is None:
|
||||
base = factory.open()
|
||||
else:
|
||||
raise ValueError("Too many base storages defined!")
|
||||
|
||||
from ZODB.DemoStorage import DemoStorage
|
||||
return DemoStorage(self.config.name, base=base, changes=changes)
|
||||
|
||||
class FileStorage(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
from ZODB.FileStorage import FileStorage
|
||||
config = self.config
|
||||
options = {}
|
||||
if getattr(config, 'packer', None):
|
||||
packer = config.packer
|
||||
if ':' in packer:
|
||||
m, expr = packer.split(':', 1)
|
||||
m = __import__(m, {}, {}, ['*'])
|
||||
options['packer'] = eval(expr, m.__dict__)
|
||||
else:
|
||||
m, name = config.packer.rsplit('.', 1)
|
||||
m = __import__(m, {}, {}, ['*'])
|
||||
options['packer'] = getattr(m, name)
|
||||
|
||||
for name in ('blob_dir', 'create', 'read_only', 'quota', 'pack_gc',
|
||||
'pack_keep_old'):
|
||||
v = getattr(config, name, self)
|
||||
if v is not self:
|
||||
options[name] = v
|
||||
|
||||
return FileStorage(config.path, **options)
|
||||
|
||||
class BlobStorage(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
from ZODB.blob import BlobStorage
|
||||
base = self.config.base.open()
|
||||
return BlobStorage(self.config.blob_dir, base)
|
||||
|
||||
|
||||
class ZEOClient(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
from ZEO.ClientStorage import ClientStorage
|
||||
# config.server is a multikey of socket-connection-address values
|
||||
# where the value is a socket family, address tuple.
|
||||
L = [server.address for server in self.config.server]
|
||||
options = {}
|
||||
if self.config.blob_cache_size is not None:
|
||||
options['blob_cache_size'] = self.config.blob_cache_size
|
||||
if self.config.blob_cache_size_check is not None:
|
||||
options['blob_cache_size_check'] = self.config.blob_cache_size_check
|
||||
if self.config.client_label is not None:
|
||||
options['client_label'] = self.config.client_label
|
||||
|
||||
return ClientStorage(
|
||||
L,
|
||||
blob_dir=self.config.blob_dir,
|
||||
shared_blob_dir=self.config.shared_blob_dir,
|
||||
storage=self.config.storage,
|
||||
cache_size=self.config.cache_size,
|
||||
name=self.config.name,
|
||||
client=self.config.client,
|
||||
var=self.config.var,
|
||||
min_disconnect_poll=self.config.min_disconnect_poll,
|
||||
max_disconnect_poll=self.config.max_disconnect_poll,
|
||||
wait=self.config.wait,
|
||||
read_only=self.config.read_only,
|
||||
read_only_fallback=self.config.read_only_fallback,
|
||||
drop_cache_rather_verify=self.config.drop_cache_rather_verify,
|
||||
username=self.config.username,
|
||||
password=self.config.password,
|
||||
realm=self.config.realm,
|
||||
**options)
|
||||
|
||||
class BDBStorage(BaseConfig):
|
||||
|
||||
def open(self):
|
||||
from BDBStorage.BerkeleyBase import BerkeleyConfig
|
||||
storageclass = self.get_storageclass()
|
||||
bconf = BerkeleyConfig()
|
||||
for name in dir(BerkeleyConfig):
|
||||
if name.startswith('_'):
|
||||
continue
|
||||
setattr(bconf, name, getattr(self.config, name))
|
||||
return storageclass(self.config.envdir, config=bconf)
|
||||
|
||||
class BDBMinimalStorage(BDBStorage):
|
||||
|
||||
def get_storageclass(self):
|
||||
import BDBStorage.BDBMinimalStorage
|
||||
return BDBStorage.BDBMinimalStorage.BDBMinimalStorage
|
||||
|
||||
class BDBFullStorage(BDBStorage):
|
||||
|
||||
def get_storageclass(self):
|
||||
import BDBStorage.BDBFullStorage
|
||||
return BDBStorage.BDBFullStorage.BDBFullStorage
|
||||
@@ -0,0 +1,8 @@
|
||||
<schema prefix="ZODB.config">
|
||||
|
||||
<import package="ZODB"/>
|
||||
|
||||
<multisection type="ZODB.database" name="*" attribute="database"
|
||||
required="yes" />
|
||||
|
||||
</schema>
|
||||
@@ -0,0 +1,34 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
import persistent.mapping
|
||||
|
||||
class fixer(object):
|
||||
def __of__(self, parent):
|
||||
def __setstate__(state, self=parent):
|
||||
self._container=state
|
||||
del self.__setstate__
|
||||
return __setstate__
|
||||
|
||||
fixer=fixer()
|
||||
|
||||
class hack(object): pass
|
||||
hack=hack()
|
||||
|
||||
def __basicnew__():
|
||||
r=persistent.mapping.PersistentMapping()
|
||||
r.__setstate__=fixer
|
||||
return r
|
||||
|
||||
hack.__basicnew__=__basicnew__
|
||||
@@ -0,0 +1,200 @@
|
||||
=========================
|
||||
Cross-Database References
|
||||
=========================
|
||||
|
||||
Persistent references to objects in different databases within a
|
||||
multi-database are allowed.
|
||||
|
||||
Lets set up a multi-database with 2 databases:
|
||||
|
||||
>>> import ZODB.tests.util, transaction, persistent
|
||||
>>> databases = {}
|
||||
>>> db1 = ZODB.tests.util.DB(databases=databases, database_name='1')
|
||||
>>> db2 = ZODB.tests.util.DB(databases=databases, database_name='2')
|
||||
|
||||
And create a persistent object in the first database:
|
||||
|
||||
>>> tm = transaction.TransactionManager()
|
||||
>>> conn1 = db1.open(transaction_manager=tm)
|
||||
>>> p1 = MyClass()
|
||||
>>> conn1.root()['p'] = p1
|
||||
>>> tm.commit()
|
||||
|
||||
First, we get a connection to the second database. We get the second
|
||||
connection using the first connection's `get_connection` method. This
|
||||
is important. When using multiple databases, we need to make sure we
|
||||
use a consistent set of connections so that the objects in the
|
||||
connection caches are connected in a consistent manner.
|
||||
|
||||
>>> conn2 = conn1.get_connection('2')
|
||||
|
||||
Now, we'll create a second persistent object in the second database.
|
||||
We'll have a reference to the first object:
|
||||
|
||||
>>> p2 = MyClass()
|
||||
>>> conn2.root()['p'] = p2
|
||||
>>> p2.p1 = p1
|
||||
>>> tm.commit()
|
||||
|
||||
Now, let's open a separate connection to database 2. We use it to
|
||||
read `p2`, use `p2` to get to `p1`, and verify that it is in database 1:
|
||||
|
||||
>>> conn = db2.open()
|
||||
>>> p2x = conn.root()['p']
|
||||
>>> p1x = p2x.p1
|
||||
|
||||
>>> p2x is p2, p2x._p_oid == p2._p_oid, p2x._p_jar.db() is db2
|
||||
(False, True, True)
|
||||
|
||||
>>> p1x is p1, p1x._p_oid == p1._p_oid, p1x._p_jar.db() is db1
|
||||
(False, True, True)
|
||||
|
||||
It isn't valid to create references outside a multi database:
|
||||
|
||||
>>> db3 = ZODB.tests.util.DB()
|
||||
>>> conn3 = db3.open(transaction_manager=tm)
|
||||
>>> p3 = MyClass()
|
||||
>>> conn3.root()['p'] = p3
|
||||
>>> tm.commit()
|
||||
|
||||
>>> p2.p3 = p3
|
||||
>>> tm.commit() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
InvalidObjectReference:
|
||||
('Attempt to store an object from a foreign database connection',
|
||||
<Connection at ...>,
|
||||
<ZODB.tests.testcrossdatabasereferences.MyClass...>)
|
||||
|
||||
>>> tm.abort()
|
||||
|
||||
Databases for new objects
|
||||
-------------------------
|
||||
|
||||
Objects are normally added to a database by making them reachable from
|
||||
an object already in the database. This is unambiguous when there is
|
||||
only one database. With multiple databases, it is not so clear what
|
||||
happens. Consider:
|
||||
|
||||
>>> p4 = MyClass()
|
||||
>>> p1.p4 = p4
|
||||
>>> p2.p4 = p4
|
||||
|
||||
In this example, the new object is reachable from both `p1` in database
|
||||
1 and `p2` in database 2. If we commit, which database should `p4` end up
|
||||
in? This sort of ambiguity could lead to subtle bugs. For that reason,
|
||||
an error is generated if we commit changes when new objects are
|
||||
reachable from multiple databases:
|
||||
|
||||
>>> tm.commit() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
InvalidObjectReference:
|
||||
("A new object is reachable from multiple databases. Won't try to
|
||||
guess which one was correct!",
|
||||
<Connection at ...>,
|
||||
<ZODB.tests.testcrossdatabasereferences.MyClass...>)
|
||||
|
||||
>>> tm.abort()
|
||||
|
||||
To resolve this ambiguity, we can commit before an object becomes
|
||||
reachable from multiple databases.
|
||||
|
||||
>>> p4 = MyClass()
|
||||
>>> p1.p4 = p4
|
||||
>>> tm.commit()
|
||||
>>> p2.p4 = p4
|
||||
>>> tm.commit()
|
||||
>>> p4._p_jar.db().database_name
|
||||
'1'
|
||||
|
||||
This doesn't work with a savepoint:
|
||||
|
||||
>>> p5 = MyClass()
|
||||
>>> p1.p5 = p5
|
||||
>>> s = tm.savepoint()
|
||||
>>> p2.p5 = p5
|
||||
>>> tm.commit() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
InvalidObjectReference:
|
||||
("A new object is reachable from multiple databases. Won't try to guess
|
||||
which one was correct!",
|
||||
<Connection at ...>,
|
||||
<ZODB.tests.testcrossdatabasereferences.MyClass...>)
|
||||
|
||||
>>> tm.abort()
|
||||
|
||||
(Maybe it should.)
|
||||
|
||||
We can disambiguate this situation by using the connection add method
|
||||
to explicitly say what database an object belongs to:
|
||||
|
||||
>>> p5 = MyClass()
|
||||
>>> p1.p5 = p5
|
||||
>>> p2.p5 = p5
|
||||
>>> conn1.add(p5)
|
||||
>>> tm.commit()
|
||||
>>> p5._p_jar.db().database_name
|
||||
'1'
|
||||
|
||||
This the most explicit and thus the best way, when practical, to avoid
|
||||
the ambiguity.
|
||||
|
||||
Dissallowing implicit cross-database references
|
||||
-----------------------------------------------
|
||||
|
||||
The database contructor accepts a xrefs keyword argument that defaults
|
||||
to True. If False is passed, the implicit cross database references
|
||||
are disallowed. (Note that currently, implicit cross references are
|
||||
the only kind of cross references allowed.)
|
||||
|
||||
>>> databases = {}
|
||||
>>> db1 = ZODB.tests.util.DB(databases=databases, database_name='1')
|
||||
>>> db2 = ZODB.tests.util.DB(databases=databases, database_name='2',
|
||||
... xrefs=False)
|
||||
|
||||
In this example, we allow cross-references from db1 to db2, but not
|
||||
the other way around.
|
||||
|
||||
>>> c1 = db1.open()
|
||||
>>> c2 = c1.get_connection('2')
|
||||
>>> c1.root.x = c2.root()
|
||||
>>> transaction.commit()
|
||||
>>> c2.root.x = c1.root()
|
||||
>>> transaction.commit() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
InvalidObjectReference:
|
||||
("Database '2' doesn't allow implicit cross-database references",
|
||||
<Connection at ...>,
|
||||
{'x': {}})
|
||||
|
||||
>>> transaction.abort()
|
||||
|
||||
NOTE
|
||||
----
|
||||
|
||||
This implementation is incomplete. It allows creating and using
|
||||
cross-database references, however, there are a number of facilities
|
||||
missing:
|
||||
|
||||
cross-database garbage collection
|
||||
|
||||
Garbage collection is done on a database by database basis.
|
||||
If an object on a database only has references to it from other
|
||||
databases, then the object will be garbage collected when its
|
||||
database is packed. The cross-database references to it will be
|
||||
broken.
|
||||
|
||||
cross-database undo
|
||||
|
||||
Undo is only applied to a single database. Fixing this for
|
||||
multiple databases is going to be extremely difficult. Undo
|
||||
currently poses consistency problems, so it is not (or should not
|
||||
be) widely used.
|
||||
|
||||
Cross-database aware (tolerant) export/import
|
||||
|
||||
The export/import facility needs to be aware, at least, of cross-database
|
||||
references.
|
||||
@@ -0,0 +1,18 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
try:
|
||||
from zope.event import notify
|
||||
except ImportError:
|
||||
notify = lambda event: None
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
Event support
|
||||
|
||||
Sometimes, you want to react when ZODB does certain things. In the
|
||||
past, ZODB provided ad hoc hook functions for this. Going forward,
|
||||
ZODB will use an event mechanism. ZODB.event.notify is called with
|
||||
events of interest.
|
||||
|
||||
If zope.event is installed, then ZODB.event.notify is simply an alias
|
||||
for zope.event. If zope.event isn't installed, then ZODB.event is a
|
||||
noop.
|
||||
@@ -0,0 +1,280 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Implement an OID to File-position (long integer) mapping."""
|
||||
|
||||
# To save space, we do two things:
|
||||
#
|
||||
# 1. We split the keys (OIDS) into 6-byte prefixes and 2-byte suffixes.
|
||||
# We use the prefixes as keys in a mapping from prefix to mappings
|
||||
# of suffix to data:
|
||||
#
|
||||
# data is {prefix -> {suffix -> data}}
|
||||
#
|
||||
# 2. We limit the data size to 48 bits. This should allow databases
|
||||
# as large as 256 terabytes.
|
||||
#
|
||||
# Most of the space is consumed by items in the mappings from 2-byte
|
||||
# suffix to 6-byte data. This should reduce the overall memory usage to
|
||||
# 8-16 bytes per OID.
|
||||
#
|
||||
# Because
|
||||
# - the mapping from suffix to data contains at most 65535 entries,
|
||||
# - this is an in-memory data structure
|
||||
# - new keys are inserted sequentially,
|
||||
# we use a BTree bucket instead of a full BTree to store the results.
|
||||
#
|
||||
# We use p64 to convert integers to 8-byte strings and lop off the two
|
||||
# high-order bytes when saving. On loading data, we add the leading
|
||||
# bytes back before using u64 to convert the data back to (long)
|
||||
# integers.
|
||||
import struct
|
||||
|
||||
from BTrees.fsBTree import fsBucket
|
||||
from BTrees.OOBTree import OOBTree
|
||||
import six
|
||||
|
||||
from ZODB._compat import INT_TYPES
|
||||
from ZODB._compat import Pickler
|
||||
from ZODB._compat import Unpickler
|
||||
from ZODB._compat import _protocol
|
||||
|
||||
|
||||
# convert between numbers and six-byte strings
|
||||
|
||||
def num2str(n):
|
||||
return struct.pack(">Q", n)[2:]
|
||||
|
||||
def str2num(s):
|
||||
return struct.unpack(">Q", b"\000\000" + s)[0]
|
||||
|
||||
def prefix_plus_one(s):
|
||||
num = str2num(s)
|
||||
return num2str(num + 1)
|
||||
|
||||
def prefix_minus_one(s):
|
||||
num = str2num(s)
|
||||
return num2str(num - 1)
|
||||
|
||||
def ensure_bytes(s):
|
||||
# on Python 3 we might pickle bytes and unpickle unicode strings
|
||||
return s.encode('ascii') if not isinstance(s, bytes) else s
|
||||
|
||||
|
||||
class fsIndex(object):
|
||||
|
||||
def __init__(self, data=None):
|
||||
self._data = OOBTree()
|
||||
if data:
|
||||
self.update(data)
|
||||
|
||||
def __getstate__(self):
|
||||
return dict(
|
||||
state_version = 1,
|
||||
_data = [(k, v.toString())
|
||||
for (k, v) in six.iteritems(self._data)
|
||||
]
|
||||
)
|
||||
|
||||
def __setstate__(self, state):
|
||||
version = state.pop('state_version', 0)
|
||||
getattr(self, '_setstate_%s' % version)(state)
|
||||
|
||||
def _setstate_0(self, state):
|
||||
self.__dict__.clear()
|
||||
self.__dict__.update(state)
|
||||
self._data = OOBTree([
|
||||
(ensure_bytes(k), v)
|
||||
for (k, v) in self._data.items()
|
||||
])
|
||||
|
||||
def _setstate_1(self, state):
|
||||
self._data = OOBTree([
|
||||
(ensure_bytes(k), fsBucket().fromString(ensure_bytes(v)))
|
||||
for (k, v) in state['_data']
|
||||
])
|
||||
|
||||
def __getitem__(self, key):
|
||||
assert isinstance(key, bytes)
|
||||
return str2num(self._data[key[:6]][key[6:]])
|
||||
|
||||
def save(self, pos, fname):
|
||||
with open(fname, 'wb') as f:
|
||||
pickler = Pickler(f, _protocol)
|
||||
pickler.fast = True
|
||||
pickler.dump(pos)
|
||||
for k, v in six.iteritems(self._data):
|
||||
pickler.dump((k, v.toString()))
|
||||
pickler.dump(None)
|
||||
|
||||
@classmethod
|
||||
def load(class_, fname):
|
||||
with open(fname, 'rb') as f:
|
||||
unpickler = Unpickler(f)
|
||||
pos = unpickler.load()
|
||||
if not isinstance(pos, INT_TYPES):
|
||||
# NB: this might contain OIDs that got unpickled
|
||||
# into Unicode strings on Python 3; hope the caller
|
||||
# will pipe the result to fsIndex().update() to normalize
|
||||
# the keys
|
||||
return pos # Old format
|
||||
index = class_()
|
||||
data = index._data
|
||||
while 1:
|
||||
v = unpickler.load()
|
||||
if not v:
|
||||
break
|
||||
k, v = v
|
||||
data[ensure_bytes(k)] = fsBucket().fromString(ensure_bytes(v))
|
||||
return dict(pos=pos, index=index)
|
||||
|
||||
def get(self, key, default=None):
|
||||
assert isinstance(key, bytes)
|
||||
tree = self._data.get(key[:6], default)
|
||||
if tree is default:
|
||||
return default
|
||||
v = tree.get(key[6:], default)
|
||||
if v is default:
|
||||
return default
|
||||
return str2num(v)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
assert isinstance(key, bytes)
|
||||
value = num2str(value)
|
||||
treekey = key[:6]
|
||||
tree = self._data.get(treekey)
|
||||
if tree is None:
|
||||
tree = fsBucket()
|
||||
self._data[treekey] = tree
|
||||
tree[key[6:]] = value
|
||||
|
||||
def __delitem__(self, key):
|
||||
assert isinstance(key, bytes)
|
||||
treekey = key[:6]
|
||||
tree = self._data.get(treekey)
|
||||
if tree is None:
|
||||
raise KeyError(key)
|
||||
del tree[key[6:]]
|
||||
if not tree:
|
||||
del self._data[treekey]
|
||||
|
||||
def __len__(self):
|
||||
r = 0
|
||||
for tree in six.itervalues(self._data):
|
||||
r += len(tree)
|
||||
return r
|
||||
|
||||
def update(self, mapping):
|
||||
for k, v in mapping.items():
|
||||
self[ensure_bytes(k)] = v
|
||||
|
||||
def has_key(self, key):
|
||||
v = self.get(key, self)
|
||||
return v is not self
|
||||
|
||||
def __contains__(self, key):
|
||||
assert isinstance(key, bytes)
|
||||
tree = self._data.get(key[:6])
|
||||
if tree is None:
|
||||
return False
|
||||
v = tree.get(key[6:], None)
|
||||
if v is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
def clear(self):
|
||||
self._data.clear()
|
||||
|
||||
def __iter__(self):
|
||||
for prefix, tree in six.iteritems(self._data):
|
||||
for suffix in tree:
|
||||
yield prefix + suffix
|
||||
|
||||
iterkeys = __iter__
|
||||
|
||||
def keys(self):
|
||||
return list(self.iterkeys())
|
||||
|
||||
def iteritems(self):
|
||||
for prefix, tree in six.iteritems(self._data):
|
||||
for suffix, value in six.iteritems(tree):
|
||||
yield (prefix + suffix, str2num(value))
|
||||
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
def itervalues(self):
|
||||
for tree in six.itervalues(self._data):
|
||||
for value in six.itervalues(tree):
|
||||
yield str2num(value)
|
||||
|
||||
def values(self):
|
||||
return list(self.itervalues())
|
||||
|
||||
# Comment below applies for the following minKey and maxKey methods
|
||||
#
|
||||
# Obscure: what if `tree` is actually empty? We're relying here on
|
||||
# that this class doesn't implement __delitem__: once a key gets
|
||||
# into an fsIndex, the only way it can go away is by invoking
|
||||
# clear(). Therefore nothing in _data.values() is ever empty.
|
||||
#
|
||||
# Note that because `tree` is an fsBTree, its minKey()/maxKey() methods are
|
||||
# very efficient.
|
||||
|
||||
def minKey(self, key=None):
|
||||
if key is None:
|
||||
smallest_prefix = self._data.minKey()
|
||||
else:
|
||||
smallest_prefix = self._data.minKey(key[:6])
|
||||
|
||||
tree = self._data[smallest_prefix]
|
||||
|
||||
assert tree
|
||||
|
||||
if key is None:
|
||||
smallest_suffix = tree.minKey()
|
||||
else:
|
||||
try:
|
||||
smallest_suffix = tree.minKey(key[6:])
|
||||
except ValueError: # 'empty tree' (no suffix >= arg)
|
||||
next_prefix = prefix_plus_one(smallest_prefix)
|
||||
smallest_prefix = self._data.minKey(next_prefix)
|
||||
tree = self._data[smallest_prefix]
|
||||
assert tree
|
||||
smallest_suffix = tree.minKey()
|
||||
|
||||
return smallest_prefix + smallest_suffix
|
||||
|
||||
def maxKey(self, key=None):
|
||||
if key is None:
|
||||
biggest_prefix = self._data.maxKey()
|
||||
else:
|
||||
biggest_prefix = self._data.maxKey(key[:6])
|
||||
|
||||
tree = self._data[biggest_prefix]
|
||||
|
||||
assert tree
|
||||
|
||||
if key is None:
|
||||
biggest_suffix = tree.maxKey()
|
||||
else:
|
||||
try:
|
||||
biggest_suffix = tree.maxKey(key[6:])
|
||||
except ValueError: # 'empty tree' (no suffix <= arg)
|
||||
next_prefix = prefix_minus_one(biggest_prefix)
|
||||
biggest_prefix = self._data.maxKey(next_prefix)
|
||||
tree = self._data[biggest_prefix]
|
||||
assert tree
|
||||
biggest_suffix = tree.maxKey()
|
||||
|
||||
return biggest_prefix + biggest_suffix
|
||||
@@ -0,0 +1,393 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Simple script for repairing damaged FileStorage files.
|
||||
|
||||
Usage: %s [-f] [-v level] [-p] [-P seconds] input output
|
||||
|
||||
Recover data from a FileStorage data file, skipping over damaged data. Any
|
||||
damaged data will be lost. This could lead to useless output if critical
|
||||
data is lost.
|
||||
|
||||
Options:
|
||||
|
||||
-f
|
||||
Overwrite output file even if it exists.
|
||||
|
||||
-v level
|
||||
|
||||
Set the verbosity level:
|
||||
|
||||
0 -- show progress indicator (default)
|
||||
|
||||
1 -- show transaction times and sizes
|
||||
|
||||
2 -- show transaction times and sizes, and show object (record)
|
||||
ids, versions, and sizes
|
||||
|
||||
-p
|
||||
|
||||
Copy partial transactions. If a data record in the middle of a
|
||||
transaction is bad, the data up to the bad data are packed. The
|
||||
output record is marked as packed. If this option is not used,
|
||||
transactions with any bad data are skipped.
|
||||
|
||||
-P t
|
||||
|
||||
Pack data to t seconds in the past. Note that if the "-p" option is
|
||||
used, then t should be 0.
|
||||
|
||||
|
||||
Important: The ZODB package must be importable. You may need to adjust
|
||||
PYTHONPATH accordingly.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
# Algorithm:
|
||||
#
|
||||
# position to start of input
|
||||
# while 1:
|
||||
# if end of file:
|
||||
# break
|
||||
# try:
|
||||
# copy_transaction
|
||||
# except:
|
||||
# scan for transaction
|
||||
# continue
|
||||
|
||||
import sys
|
||||
import os
|
||||
import getopt
|
||||
import time
|
||||
from struct import unpack
|
||||
|
||||
try:
|
||||
import ZODB
|
||||
except ImportError:
|
||||
if os.path.exists('ZODB'):
|
||||
sys.path.append('.')
|
||||
elif os.path.exists('FileStorage.py'):
|
||||
sys.path.append('..')
|
||||
import ZODB
|
||||
|
||||
import ZODB.FileStorage
|
||||
from ZODB.utils import u64, as_text
|
||||
from ZODB.FileStorage import TransactionRecord
|
||||
from ZODB._compat import loads
|
||||
|
||||
from persistent.TimeStamp import TimeStamp
|
||||
|
||||
|
||||
def die(mess='', show_docstring=False):
|
||||
if mess:
|
||||
print(mess + '\n', file=sys.stderr)
|
||||
if show_docstring:
|
||||
print(__doc__ % sys.argv[0], file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
class ErrorFound(Exception):
|
||||
pass
|
||||
|
||||
def error(mess, *args):
|
||||
raise ErrorFound(mess % args)
|
||||
|
||||
def read_txn_header(f, pos, file_size, outp, ltid):
|
||||
# Read the transaction record
|
||||
f.seek(pos)
|
||||
h = f.read(23)
|
||||
if len(h) < 23:
|
||||
raise EOFError
|
||||
|
||||
tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h)
|
||||
status = as_text(status)
|
||||
tl = u64(stl)
|
||||
|
||||
if pos + (tl + 8) > file_size:
|
||||
error("bad transaction length at %s", pos)
|
||||
|
||||
if tl < (23 + ul + dl + el):
|
||||
error("invalid transaction length, %s, at %s", tl, pos)
|
||||
|
||||
if ltid and tid < ltid:
|
||||
error("time-stamp reducation %s < %s, at %s", u64(tid), u64(ltid), pos)
|
||||
|
||||
if status == "c":
|
||||
truncate(f, pos, file_size, outp)
|
||||
raise EOFError
|
||||
|
||||
if status not in " up":
|
||||
error("invalid status, %r, at %s", status, pos)
|
||||
|
||||
tpos = pos
|
||||
tend = tpos + tl
|
||||
|
||||
if status == "u":
|
||||
# Undone transaction, skip it
|
||||
f.seek(tend)
|
||||
h = f.read(8)
|
||||
if h != stl:
|
||||
error("inconsistent transaction length at %s", pos)
|
||||
pos = tend + 8
|
||||
return pos, None, tid
|
||||
|
||||
pos = tpos+(23+ul+dl+el)
|
||||
user = f.read(ul)
|
||||
description = f.read(dl)
|
||||
if el:
|
||||
try: e = loads(f.read(el))
|
||||
except: e = {}
|
||||
else: e = {}
|
||||
|
||||
result = TransactionRecord(tid, status, user, description, e, pos, tend,
|
||||
f, tpos)
|
||||
pos = tend
|
||||
|
||||
# Read the (intentionally redundant) transaction length
|
||||
f.seek(pos)
|
||||
h = f.read(8)
|
||||
if h != stl:
|
||||
error("redundant transaction length check failed at %s", pos)
|
||||
pos += 8
|
||||
|
||||
return pos, result, tid
|
||||
|
||||
def truncate(f, pos, file_size, outp):
|
||||
"""Copy data from pos to end of f to a .trNNN file."""
|
||||
|
||||
# _trname is global so that the test suite can know the path too (in
|
||||
# order to delete the file when the test ends).
|
||||
global _trname
|
||||
|
||||
i = 0
|
||||
while 1:
|
||||
_trname = outp + ".tr%d" % i
|
||||
if os.path.exists(_trname):
|
||||
i += 1
|
||||
else:
|
||||
break
|
||||
tr = open(_trname, "wb")
|
||||
copy(f, tr, file_size - pos)
|
||||
f.seek(pos)
|
||||
tr.close()
|
||||
|
||||
def copy(src, dst, n):
|
||||
while n:
|
||||
buf = src.read(8096)
|
||||
if not buf:
|
||||
break
|
||||
if len(buf) > n:
|
||||
buf = buf[:n]
|
||||
dst.write(buf)
|
||||
n -= len(buf)
|
||||
|
||||
def scan(f, pos):
|
||||
"""Return a potential transaction location following pos in f.
|
||||
|
||||
This routine scans forward from pos looking for the last data
|
||||
record in a transaction. A period '.' always occurs at the end of
|
||||
a pickle, and an 8-byte transaction length follows the last
|
||||
pickle. If a period is followed by a plausible 8-byte transaction
|
||||
length, assume that we have found the end of a transaction.
|
||||
|
||||
The caller should try to verify that the returned location is
|
||||
actually a transaction header.
|
||||
"""
|
||||
while 1:
|
||||
f.seek(pos)
|
||||
data = f.read(8096)
|
||||
if not data:
|
||||
return 0
|
||||
|
||||
s = 0
|
||||
while 1:
|
||||
l = data.find(b".", s)
|
||||
if l < 0:
|
||||
pos += len(data)
|
||||
break
|
||||
# If we are less than 8 bytes from the end of the
|
||||
# string, we need to read more data.
|
||||
s = l + 1
|
||||
if s > len(data) - 8:
|
||||
pos += l
|
||||
break
|
||||
tl = u64(data[s:s+8])
|
||||
if tl < pos:
|
||||
return pos + s + 8
|
||||
|
||||
def iprogress(i):
|
||||
if i % 2:
|
||||
print(".", end=' ')
|
||||
else:
|
||||
print((i/2) % 10, end=' ')
|
||||
sys.stdout.flush()
|
||||
|
||||
def progress(p):
|
||||
for i in range(p):
|
||||
iprogress(i)
|
||||
|
||||
def main():
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "fv:pP:")
|
||||
except getopt.error as msg:
|
||||
die(str(msg), show_docstring=True)
|
||||
|
||||
if len(args) != 2:
|
||||
die("two positional arguments required", show_docstring=True)
|
||||
inp, outp = args
|
||||
|
||||
force = partial = False
|
||||
verbose = 0
|
||||
pack = None
|
||||
for opt, v in opts:
|
||||
if opt == "-v":
|
||||
verbose = int(v)
|
||||
elif opt == "-p":
|
||||
partial = True
|
||||
elif opt == "-f":
|
||||
force = True
|
||||
elif opt == "-P":
|
||||
pack = time.time() - float(v)
|
||||
|
||||
recover(inp, outp, verbose, partial, force, pack)
|
||||
|
||||
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
|
||||
print("Recovering", inp, "into", outp)
|
||||
|
||||
if os.path.exists(outp) and not force:
|
||||
die("%s exists" % outp)
|
||||
|
||||
f = open(inp, "rb")
|
||||
if f.read(4) != ZODB.FileStorage.packed_version:
|
||||
die("input is not a file storage")
|
||||
|
||||
f.seek(0,2)
|
||||
file_size = f.tell()
|
||||
|
||||
ofs = ZODB.FileStorage.FileStorage(outp, create=1)
|
||||
_ts = None
|
||||
ok = 1
|
||||
prog1 = 0
|
||||
undone = 0
|
||||
|
||||
pos = 4
|
||||
ltid = None
|
||||
while pos:
|
||||
try:
|
||||
npos, txn, tid = read_txn_header(f, pos, file_size, outp, ltid)
|
||||
except EOFError:
|
||||
break
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except Exception as err:
|
||||
print("error reading txn header:", err)
|
||||
if not verbose:
|
||||
progress(prog1)
|
||||
pos = scan(f, pos)
|
||||
if verbose > 1:
|
||||
print("looking for valid txn header at", pos)
|
||||
continue
|
||||
ltid = tid
|
||||
|
||||
if txn is None:
|
||||
undone = undone + npos - pos
|
||||
pos = npos
|
||||
continue
|
||||
else:
|
||||
pos = npos
|
||||
|
||||
tid = txn.tid
|
||||
|
||||
if _ts is None:
|
||||
_ts = TimeStamp(tid)
|
||||
else:
|
||||
t = TimeStamp(tid)
|
||||
if t <= _ts:
|
||||
if ok:
|
||||
print(("Time stamps out of order %s, %s" % (_ts, t)))
|
||||
ok = 0
|
||||
_ts = t.laterThan(_ts)
|
||||
tid = _ts.raw()
|
||||
else:
|
||||
_ts = t
|
||||
if not ok:
|
||||
print(("Time stamps back in order %s" % (t)))
|
||||
ok = 1
|
||||
|
||||
ofs.tpc_begin(txn, tid, txn.status)
|
||||
|
||||
if verbose:
|
||||
print("begin", pos, _ts, end=' ')
|
||||
if verbose > 1:
|
||||
print()
|
||||
sys.stdout.flush()
|
||||
|
||||
nrec = 0
|
||||
try:
|
||||
for r in txn:
|
||||
if verbose > 1:
|
||||
if r.data is None:
|
||||
l = "bp"
|
||||
else:
|
||||
l = len(r.data)
|
||||
|
||||
print("%7d %s %s" % (u64(r.oid), l))
|
||||
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
|
||||
txn)
|
||||
nrec += 1
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except Exception as err:
|
||||
if partial and nrec:
|
||||
ofs._status = "p"
|
||||
ofs.tpc_vote(txn)
|
||||
ofs.tpc_finish(txn)
|
||||
if verbose:
|
||||
print("partial")
|
||||
else:
|
||||
ofs.tpc_abort(txn)
|
||||
print("error copying transaction:", err)
|
||||
if not verbose:
|
||||
progress(prog1)
|
||||
pos = scan(f, pos)
|
||||
if verbose > 1:
|
||||
print("looking for valid txn header at", pos)
|
||||
else:
|
||||
ofs.tpc_vote(txn)
|
||||
ofs.tpc_finish(txn)
|
||||
if verbose:
|
||||
print("finish")
|
||||
sys.stdout.flush()
|
||||
|
||||
if not verbose:
|
||||
prog = pos * 20 / file_size
|
||||
while prog > prog1:
|
||||
prog1 = prog1 + 1
|
||||
iprogress(prog1)
|
||||
|
||||
|
||||
bad = file_size - undone - ofs._pos
|
||||
|
||||
print("\n%s bytes removed during recovery" % bad)
|
||||
if undone:
|
||||
print("%s bytes of undone transaction data were skipped" % undone)
|
||||
|
||||
if pack is not None:
|
||||
print("Packing ...")
|
||||
from ZODB.serialize import referencesf
|
||||
ofs.pack(pack, referencesf)
|
||||
|
||||
ofs.close()
|
||||
f.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
"""Tools for using FileStorage data files.
|
||||
|
||||
TODO: This module needs tests.
|
||||
Caution: This file needs to be kept in sync with FileStorage.py.
|
||||
"""
|
||||
|
||||
import struct
|
||||
|
||||
from ZODB.FileStorage.format import TRANS_HDR, DATA_HDR, TRANS_HDR_LEN
|
||||
from ZODB.FileStorage.format import DATA_HDR_LEN
|
||||
from ZODB.utils import u64
|
||||
from ZODB._compat import loads
|
||||
from persistent.TimeStamp import TimeStamp
|
||||
|
||||
|
||||
class TxnHeader(object):
|
||||
"""Object representing a transaction record header.
|
||||
|
||||
Attribute Position Value
|
||||
--------- -------- -----
|
||||
tid 0- 8 transaction id
|
||||
length 8-16 length of entire transaction record - 8
|
||||
status 16-17 status of transaction (' ', 'u', 'p'?)
|
||||
user_len 17-19 length of user field (pack code H)
|
||||
descr_len 19-21 length of description field (pack code H)
|
||||
ext_len 21-23 length of extensions (pack code H)
|
||||
"""
|
||||
|
||||
def __init__(self, file, pos):
|
||||
self._file = file
|
||||
self._pos = pos
|
||||
self._read_header()
|
||||
|
||||
def _read_header(self):
|
||||
self._file.seek(self._pos)
|
||||
self._hdr = self._file.read(TRANS_HDR_LEN)
|
||||
(self.tid, self.length, self.status, self.user_len, self.descr_len,
|
||||
self.ext_len) = struct.unpack(TRANS_HDR, self._hdr)
|
||||
|
||||
def read_meta(self):
|
||||
"""Load user, descr, and ext attributes."""
|
||||
self.user = ""
|
||||
self.descr = ""
|
||||
self.ext = {}
|
||||
if not (self.user_len or self.descr_len or self.ext_len):
|
||||
return
|
||||
self._file.seek(self._pos + TRANS_HDR_LEN)
|
||||
if self.user_len:
|
||||
self.user = self._file.read(self.user_len)
|
||||
if self.descr_len:
|
||||
self.descr = self._file.read(self.descr_len)
|
||||
if self.ext_len:
|
||||
self._ext = self._file.read(self.ext_len)
|
||||
self.ext = loads(self._ext)
|
||||
|
||||
def get_offset(self):
|
||||
return self._pos
|
||||
|
||||
def __len__(self):
|
||||
return TRANS_HDR_LEN + self.user_len + self.descr_len + self.ext_len
|
||||
|
||||
def get_data_offset(self):
|
||||
return self._pos + len(self)
|
||||
|
||||
def get_timestamp(self):
|
||||
return TimeStamp(self.tid)
|
||||
|
||||
def get_raw_data(self):
|
||||
data_off = self.get_data_offset()
|
||||
data_len = self.length - (data_off - self._pos)
|
||||
self._file.seek(data_off)
|
||||
return self._file.read(data_len)
|
||||
|
||||
def next_txn(self):
|
||||
off = self._pos + self.length + 8
|
||||
self._file.seek(off)
|
||||
s = self._file.read(8)
|
||||
if not s:
|
||||
return None
|
||||
return TxnHeader(self._file, off)
|
||||
|
||||
def prev_txn(self):
|
||||
if self._pos == 4:
|
||||
return None
|
||||
self._file.seek(self._pos - 8)
|
||||
tlen = u64(self._file.read(8))
|
||||
return TxnHeader(self._file, self._pos - (tlen + 8))
|
||||
|
||||
class DataHeader(object):
|
||||
"""Object representing a data record header.
|
||||
|
||||
Attribute Position Value
|
||||
--------- -------- -----
|
||||
oid 0- 8 object id
|
||||
serial 8-16 object serial numver
|
||||
prev_rec_pos 16-24 position of previous data record for object
|
||||
txn_pos 24-32 position of txn header
|
||||
version_len 32-34 length of version (always 0)
|
||||
data_len 34-42 length of data
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, file, pos):
|
||||
self._file = file
|
||||
self._pos = pos
|
||||
self._read_header()
|
||||
|
||||
def _read_header(self):
|
||||
self._file.seek(self._pos)
|
||||
self._hdr = self._file.read(DATA_HDR_LEN)
|
||||
# always read the longer header, just in case
|
||||
(self.oid, self.serial, prev_rec_pos, txn_pos, vlen, data_len
|
||||
) = struct.unpack(DATA_HDR, self._hdr[:DATA_HDR_LEN])
|
||||
assert not vlen
|
||||
self.prev_rec_pos = u64(prev_rec_pos)
|
||||
self.txn_pos = u64(txn_pos)
|
||||
self.data_len = u64(data_len)
|
||||
|
||||
def next_offset(self):
|
||||
"""Return offset of next record."""
|
||||
off = self._pos + self.data_len
|
||||
off += DATA_HDR_LEN
|
||||
if self.data_len == 0:
|
||||
off += 8 # backpointer
|
||||
return off
|
||||
|
||||
def prev_txn(f):
|
||||
"""Return transaction located before current file position."""
|
||||
f.seek(-8, 1)
|
||||
tlen = u64(f.read(8)) + 8
|
||||
return TxnHeader(f, f.tell() - tlen)
|
||||
@@ -0,0 +1,332 @@
|
||||
======================
|
||||
Historical Connections
|
||||
======================
|
||||
|
||||
.. We need to mess with time to prevent spurious test failures on windows
|
||||
|
||||
>>> _now = 1231019584.0
|
||||
>>> def faux_time_time():
|
||||
... global _now
|
||||
... _now += .001 # must be less than 0.01
|
||||
... return _now
|
||||
>>> import time
|
||||
>>> real_time_time = time.time
|
||||
>>> real_time_sleep = time.sleep
|
||||
>>> def faux_time_sleep(amt):
|
||||
... global _now
|
||||
... _now += amt
|
||||
>>> if isinstance(time,type):
|
||||
... time.time = staticmethod(faux_time_time) # Jython
|
||||
... time.sleep = faux_time_sleep
|
||||
... else:
|
||||
... time.time = faux_time_time
|
||||
... time.sleep = faux_time_sleep
|
||||
>>> def utcnow():
|
||||
... mus = (int(_now % 1 * 1000000), )
|
||||
... return datetime.datetime(*time.gmtime(_now)[:6] + mus)
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
A database can be opened with a read-only, historical connection when given
|
||||
a specific transaction or datetime. This can enable full-context application
|
||||
level conflict resolution, historical exploration and preparation for reverts,
|
||||
or even the use of a historical database revision as "production" while
|
||||
development continues on a "development" head.
|
||||
|
||||
A database can be opened historically ``at`` or ``before`` a given transaction
|
||||
serial or datetime. Here's a simple example. It should work with any storage
|
||||
that supports ``loadBefore``.
|
||||
|
||||
We'll begin our example with a fairly standard set up. We
|
||||
|
||||
- make a storage and a database;
|
||||
- open a normal connection;
|
||||
- modify the database through the connection;
|
||||
- commit a transaction, remembering the time in UTC;
|
||||
- modify the database again; and
|
||||
- commit a transaction.
|
||||
|
||||
>>> import ZODB.MappingStorage
|
||||
>>> db = ZODB.MappingStorage.DB()
|
||||
>>> conn = db.open()
|
||||
|
||||
>>> import persistent.mapping
|
||||
|
||||
>>> conn.root()['first'] = persistent.mapping.PersistentMapping(count=0)
|
||||
|
||||
>>> import transaction
|
||||
>>> transaction.commit()
|
||||
|
||||
We wait for some time to pass, record he time, and then make some other changes.
|
||||
|
||||
>>> import time
|
||||
>>> time.sleep(.01)
|
||||
|
||||
>>> import datetime
|
||||
>>> now = utcnow()
|
||||
>>> time.sleep(.01)
|
||||
|
||||
>>> root = conn.root()
|
||||
>>> root['second'] = persistent.mapping.PersistentMapping()
|
||||
>>> root['first']['count'] += 1
|
||||
|
||||
>>> transaction.commit()
|
||||
|
||||
Now we will show a historical connection. We'll open one using the ``now``
|
||||
value we generated above, and then demonstrate that the state of the original
|
||||
connection, at the mutable head of the database, is different than the
|
||||
historical state.
|
||||
|
||||
>>> transaction1 = transaction.TransactionManager()
|
||||
|
||||
>>> historical_conn = db.open(transaction_manager=transaction1, at=now)
|
||||
|
||||
>>> sorted(conn.root().keys())
|
||||
['first', 'second']
|
||||
>>> conn.root()['first']['count']
|
||||
1
|
||||
|
||||
>>> sorted(historical_conn.root().keys())
|
||||
['first']
|
||||
>>> historical_conn.root()['first']['count']
|
||||
0
|
||||
|
||||
Moreover, the historical connection cannot commit changes.
|
||||
|
||||
>>> historical_conn.root()['first']['count'] += 1
|
||||
>>> historical_conn.root()['first']['count']
|
||||
1
|
||||
>>> transaction1.commit()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ReadOnlyHistoryError
|
||||
>>> transaction1.abort()
|
||||
>>> historical_conn.root()['first']['count']
|
||||
0
|
||||
|
||||
(It is because of the mutable behavior outside of transactional semantics that
|
||||
we must have a separate connection, and associated object cache, per thread,
|
||||
even though the semantics should be readonly.)
|
||||
|
||||
As demonstrated, a timezone-naive datetime will be interpreted as UTC. You
|
||||
can also pass a timezone-aware datetime or a serial (transaction id).
|
||||
Here's opening with a serial--the serial of the root at the time of the first
|
||||
commit.
|
||||
|
||||
>>> historical_serial = historical_conn.root()._p_serial
|
||||
>>> historical_conn.close()
|
||||
|
||||
>>> historical_conn = db.open(transaction_manager=transaction1,
|
||||
... at=historical_serial)
|
||||
>>> sorted(historical_conn.root().keys())
|
||||
['first']
|
||||
>>> historical_conn.root()['first']['count']
|
||||
0
|
||||
>>> historical_conn.close()
|
||||
|
||||
We've shown the ``at`` argument. You can also ask to look ``before`` a datetime
|
||||
or serial. (It's an error to pass both [#not_both]_) In this example, we're
|
||||
looking at the database immediately prior to the most recent change to the
|
||||
root.
|
||||
|
||||
>>> serial = conn.root()._p_serial
|
||||
>>> historical_conn = db.open(
|
||||
... transaction_manager=transaction1, before=serial)
|
||||
>>> sorted(historical_conn.root().keys())
|
||||
['first']
|
||||
>>> historical_conn.root()['first']['count']
|
||||
0
|
||||
|
||||
In fact, ``at`` arguments are translated into ``before`` values because the
|
||||
underlying mechanism is a storage's loadBefore method. When you look at a
|
||||
connection's ``before`` attribute, it is normalized into a ``before`` serial,
|
||||
no matter what you pass into ``db.open``.
|
||||
|
||||
>>> print(conn.before)
|
||||
None
|
||||
>>> historical_conn.before == serial
|
||||
True
|
||||
|
||||
>>> conn.close()
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
Like normal connections, the database lets you set how many total historical
|
||||
connections can be active without generating a warning, and
|
||||
how many objects should be kept in each historical connection's object cache.
|
||||
|
||||
>>> db.getHistoricalPoolSize()
|
||||
3
|
||||
>>> db.setHistoricalPoolSize(4)
|
||||
>>> db.getHistoricalPoolSize()
|
||||
4
|
||||
|
||||
>>> db.getHistoricalCacheSize()
|
||||
1000
|
||||
>>> db.setHistoricalCacheSize(2000)
|
||||
>>> db.getHistoricalCacheSize()
|
||||
2000
|
||||
|
||||
In addition, you can specify the minimum number of seconds that an unused
|
||||
historical connection should be kept.
|
||||
|
||||
>>> db.getHistoricalTimeout()
|
||||
300
|
||||
>>> db.setHistoricalTimeout(400)
|
||||
>>> db.getHistoricalTimeout()
|
||||
400
|
||||
|
||||
All three of these values can be specified in a ZConfig file.
|
||||
|
||||
>>> import ZODB.config
|
||||
>>> db2 = ZODB.config.databaseFromString('''
|
||||
... <zodb>
|
||||
... <mappingstorage/>
|
||||
... historical-pool-size 3
|
||||
... historical-cache-size 1500
|
||||
... historical-timeout 6m
|
||||
... </zodb>
|
||||
... ''')
|
||||
>>> db2.getHistoricalPoolSize()
|
||||
3
|
||||
>>> db2.getHistoricalCacheSize()
|
||||
1500
|
||||
>>> db2.getHistoricalTimeout()
|
||||
360
|
||||
|
||||
|
||||
The pool lets us reuse connections. To see this, we'll open some
|
||||
connections, close them, and then open them again:
|
||||
|
||||
>>> conns1 = [db2.open(before=serial) for i in range(4)]
|
||||
>>> _ = [c.close() for c in conns1]
|
||||
>>> conns2 = [db2.open(before=serial) for i in range(4)]
|
||||
|
||||
Now let's look at what we got. The first connection in conns 2 is the
|
||||
last connection in conns1, because it was the last connection closed.
|
||||
|
||||
>>> conns2[0] is conns1[-1]
|
||||
True
|
||||
|
||||
Also for the next two:
|
||||
|
||||
>>> (conns2[1] is conns1[-2]), (conns2[2] is conns1[-3])
|
||||
(True, True)
|
||||
|
||||
But not for the last:
|
||||
|
||||
>>> conns2[3] is conns1[-4]
|
||||
False
|
||||
|
||||
Because the pool size was set to 3.
|
||||
|
||||
Connections are also discarded if they haven't been used in a while.
|
||||
To see this, let's close two of the connections:
|
||||
|
||||
>>> conns2[0].close(); conns2[1].close()
|
||||
|
||||
We'l also set the historical timeout to be very low:
|
||||
|
||||
>>> db2.setHistoricalTimeout(.01)
|
||||
>>> time.sleep(.1)
|
||||
>>> conns2[2].close(); conns2[3].close()
|
||||
|
||||
Now, when we open 4 connections:
|
||||
|
||||
>>> conns1 = [db2.open(before=serial) for i in range(4)]
|
||||
|
||||
We'll see that only the last 2 connections from conn2 are in the
|
||||
result:
|
||||
|
||||
>>> [c in conns1 for c in conns2]
|
||||
[False, False, True, True]
|
||||
|
||||
|
||||
If you change the historical cache size, that changes the size of the
|
||||
persistent cache on our connection.
|
||||
|
||||
>>> historical_conn._cache.cache_size
|
||||
2000
|
||||
>>> db.setHistoricalCacheSize(1500)
|
||||
>>> historical_conn._cache.cache_size
|
||||
1500
|
||||
|
||||
Invalidations
|
||||
=============
|
||||
|
||||
Invalidations are ignored for historical connections. This is another white box
|
||||
test.
|
||||
|
||||
>>> historical_conn = db.open(
|
||||
... transaction_manager=transaction1, at=serial)
|
||||
>>> conn = db.open()
|
||||
>>> sorted(conn.root().keys())
|
||||
['first', 'second']
|
||||
>>> conn.root()['first']['count']
|
||||
1
|
||||
>>> sorted(historical_conn.root().keys())
|
||||
['first', 'second']
|
||||
>>> historical_conn.root()['first']['count']
|
||||
1
|
||||
>>> conn.root()['first']['count'] += 1
|
||||
>>> conn.root()['third'] = persistent.mapping.PersistentMapping()
|
||||
>>> transaction.commit()
|
||||
>>> historical_conn.close()
|
||||
|
||||
Note that if you try to open an historical connection to a time in the future,
|
||||
you will get an error.
|
||||
|
||||
>>> historical_conn = db.open(
|
||||
... at=utcnow()+datetime.timedelta(1))
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: cannot open an historical connection in the future.
|
||||
|
||||
Warnings
|
||||
========
|
||||
|
||||
First, if you use datetimes to get a historical connection, be aware that the
|
||||
conversion from datetime to transaction id has some pitfalls. Generally, the
|
||||
transaction ids in the database are only as time-accurate as the system clock
|
||||
was when the transaction id was created. Moreover, leap seconds are handled
|
||||
somewhat naively in the ZODB (largely because they are handled naively in Unix/
|
||||
POSIX time) so any minute that contains a leap second may contain serials that
|
||||
are a bit off. This is not generally a problem for the ZODB, because serials
|
||||
are guaranteed to increase, but it does highlight the fact that serials are not
|
||||
guaranteed to be accurately connected to time. Generally, they are about as
|
||||
reliable as time.time.
|
||||
|
||||
Second, historical connections currently introduce potentially wide variance in
|
||||
memory requirements for the applications. Since you can open up many
|
||||
connections to different serials, and each gets their own pool, you may collect
|
||||
quite a few connections. For now, at least, if you use this feature you need to
|
||||
be particularly careful of your memory usage. Get rid of pools when you know
|
||||
you can, and reuse the exact same values for ``at`` or ``before`` when
|
||||
possible. If historical connections are used for conflict resolution, these
|
||||
connections will probably be temporary--not saved in a pool--so that the extra
|
||||
memory usage would also be brief and unlikely to overlap.
|
||||
|
||||
|
||||
.. cleanup
|
||||
|
||||
>>> db.close()
|
||||
>>> db2.close()
|
||||
|
||||
.. restore time
|
||||
|
||||
>>> time.time = real_time_time
|
||||
>>> time.sleep = real_time_sleep
|
||||
|
||||
.. ......... ..
|
||||
.. Footnotes ..
|
||||
.. ......... ..
|
||||
|
||||
.. [#not_both] It is an error to try and pass both `at` and `before`.
|
||||
|
||||
>>> historical_conn = db.open(
|
||||
... transaction_manager=transaction1, at=now, before=historical_serial)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: can only pass zero or one of `at` and `before`
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,47 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Supplies custom logging levels BLATHER and TRACE.
|
||||
|
||||
$Revision: 1.1 $
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
__all__ = ["BLATHER", "TRACE"]
|
||||
|
||||
# In the days of zLOG, there were 7 standard log levels, and ZODB/ZEO used
|
||||
# all of them. Here's how they map to the logging package's 5 standard
|
||||
# levels:
|
||||
#
|
||||
# zLOG logging
|
||||
# ------------- ---------------
|
||||
# PANIC (300) FATAL, CRITICAL (50)
|
||||
# ERROR (200) ERROR (40)
|
||||
# WARNING, PROBLEM (100) WARN (30)
|
||||
# INFO (0) INFO (20)
|
||||
# BLATHER (-100) none -- defined here as BLATHER (15)
|
||||
# DEBUG (-200) DEBUG (10)
|
||||
# TRACE (-300) none -- defined here as TRACE (5)
|
||||
#
|
||||
# TRACE is used by ZEO for extremely verbose trace output, enabled only
|
||||
# when chasing bottom-level communications bugs. It really should be at
|
||||
# a lower level than DEBUG.
|
||||
#
|
||||
# BLATHER is a harder call, and various instances could probably be folded
|
||||
# into INFO or DEBUG without real harm.
|
||||
|
||||
BLATHER = 15
|
||||
TRACE = 5
|
||||
logging.addLevelName(BLATHER, "BLATHER")
|
||||
logging.addLevelName(TRACE, "TRACE")
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Adapt IStorage objects to IMVCCStorage
|
||||
|
||||
This is a largely internal implementation of ZODB, especially DB and
|
||||
Connection. It takes the MVCC implementation involving invalidations
|
||||
and start time and moves it into a storage adapter. This allows ZODB
|
||||
to treat Relstoage and other storages in pretty much the same way and
|
||||
also simplifies the implementation of the DB and Connection classes.
|
||||
"""
|
||||
import zope.interface
|
||||
|
||||
from . import interfaces, serialize, POSException
|
||||
from .utils import p64, u64, Lock
|
||||
|
||||
class Base(object):
|
||||
|
||||
_copy_methods = (
|
||||
'getName', 'getSize', 'history', 'lastTransaction', 'sortKey',
|
||||
'loadBlob', 'openCommittedBlobFile',
|
||||
'isReadOnly', 'supportsUndo', 'undoLog', 'undoInfo',
|
||||
'temporaryDirectory',
|
||||
)
|
||||
|
||||
def __init__(self, storage):
|
||||
self._storage = storage
|
||||
if interfaces.IBlobStorage.providedBy(storage):
|
||||
zope.interface.alsoProvides(self, interfaces.IBlobStorage)
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name in self._copy_methods:
|
||||
m = getattr(self._storage, name)
|
||||
setattr(self, name, m)
|
||||
return m
|
||||
|
||||
raise AttributeError(name)
|
||||
|
||||
def __len__(self):
|
||||
return len(self._storage)
|
||||
|
||||
class MVCCAdapter(Base):
|
||||
|
||||
def __init__(self, storage):
|
||||
Base.__init__(self, storage)
|
||||
self._instances = set()
|
||||
self._lock = Lock()
|
||||
if hasattr(storage, 'registerDB'):
|
||||
storage.registerDB(self)
|
||||
|
||||
def new_instance(self):
|
||||
instance = MVCCAdapterInstance(self)
|
||||
with self._lock:
|
||||
self._instances.add(instance)
|
||||
return instance
|
||||
|
||||
def before_instance(self, before=None):
|
||||
return HistoricalStorageAdapter(self._storage, before)
|
||||
|
||||
def undo_instance(self):
|
||||
return UndoAdapterInstance(self)
|
||||
|
||||
def _release(self, instance):
|
||||
with self._lock:
|
||||
self._instances.remove(instance)
|
||||
|
||||
closed = False
|
||||
def close(self):
|
||||
if not self.closed:
|
||||
self.closed = True
|
||||
self._storage.close()
|
||||
del self._instances
|
||||
del self._storage
|
||||
|
||||
def invalidateCache(self):
|
||||
with self._lock:
|
||||
for instance in self._instances:
|
||||
instance._invalidateCache()
|
||||
|
||||
def invalidate(self, transaction_id, oids):
|
||||
with self._lock:
|
||||
for instance in self._instances:
|
||||
instance._invalidate(oids)
|
||||
|
||||
def _invalidate_finish(self, oids, committing_instance):
|
||||
with self._lock:
|
||||
for instance in self._instances:
|
||||
if instance is not committing_instance:
|
||||
instance._invalidate(oids)
|
||||
|
||||
references = serialize.referencesf
|
||||
transform_record_data = untransform_record_data = lambda self, data: data
|
||||
|
||||
def pack(self, pack_time, referencesf):
|
||||
return self._storage.pack(pack_time, referencesf)
|
||||
|
||||
class MVCCAdapterInstance(Base):
|
||||
|
||||
_copy_methods = Base._copy_methods + (
|
||||
'loadSerial', 'new_oid', 'tpc_vote',
|
||||
'checkCurrentSerialInTransaction', 'tpc_abort',
|
||||
)
|
||||
|
||||
def __init__(self, base):
|
||||
self._base = base
|
||||
Base.__init__(self, base._storage)
|
||||
self._lock = Lock()
|
||||
self._invalidations = set()
|
||||
self._start = None # Transaction start time
|
||||
self._sync = getattr(self._storage, 'sync', lambda : None)
|
||||
|
||||
def release(self):
|
||||
self._base._release(self)
|
||||
|
||||
close = release
|
||||
|
||||
def _invalidateCache(self):
|
||||
with self._lock:
|
||||
self._invalidations = None
|
||||
|
||||
def _invalidate(self, oids):
|
||||
with self._lock:
|
||||
try:
|
||||
self._invalidations.update(oids)
|
||||
except AttributeError:
|
||||
if self._invalidations is not None:
|
||||
raise
|
||||
|
||||
def sync(self, force=True):
|
||||
if force:
|
||||
self._sync()
|
||||
|
||||
def poll_invalidations(self):
|
||||
self._start = p64(u64(self._storage.lastTransaction()) + 1)
|
||||
with self._lock:
|
||||
if self._invalidations is None:
|
||||
self._invalidations = set()
|
||||
return None
|
||||
else:
|
||||
result = list(self._invalidations)
|
||||
self._invalidations.clear()
|
||||
return result
|
||||
|
||||
def load(self, oid):
|
||||
assert self._start is not None
|
||||
r = self._storage.loadBefore(oid, self._start)
|
||||
if r is None:
|
||||
raise POSException.ReadConflictError(repr(oid))
|
||||
return r[:2]
|
||||
|
||||
def prefetch(self, oids):
|
||||
try:
|
||||
self._storage.prefetch(oids, self._start)
|
||||
except AttributeError:
|
||||
if not hasattr(self._storage, 'prefetch'):
|
||||
self.prefetch = lambda *a: None
|
||||
else:
|
||||
raise
|
||||
|
||||
_modified = None # Used to keep track of oids modified within a
|
||||
# transaction, so we can invalidate them later.
|
||||
|
||||
def tpc_begin(self, transaction):
|
||||
self._storage.tpc_begin(transaction)
|
||||
self._modified = set()
|
||||
|
||||
def store(self, oid, serial, data, version, transaction):
|
||||
self._storage.store(oid, serial, data, version, transaction)
|
||||
self._modified.add(oid)
|
||||
|
||||
def storeBlob(self, oid, serial, data, blobfilename, version, transaction):
|
||||
self._storage.storeBlob(
|
||||
oid, serial, data, blobfilename, '', transaction)
|
||||
self._modified.add(oid)
|
||||
|
||||
def tpc_finish(self, transaction, func = lambda tid: None):
|
||||
modified = self._modified
|
||||
self._modified = None
|
||||
|
||||
def invalidate_finish(tid):
|
||||
self._base._invalidate_finish(modified, self)
|
||||
func(tid)
|
||||
|
||||
return self._storage.tpc_finish(transaction, invalidate_finish)
|
||||
|
||||
def read_only_writer(self, *a, **kw):
|
||||
raise POSException.ReadOnlyError
|
||||
|
||||
class HistoricalStorageAdapter(Base):
|
||||
"""Adapt a storage to a historical storage
|
||||
"""
|
||||
|
||||
_copy_methods = Base._copy_methods + (
|
||||
'loadSerial', 'tpc_begin', 'tpc_finish', 'tpc_abort', 'tpc_vote',
|
||||
'checkCurrentSerialInTransaction',
|
||||
)
|
||||
|
||||
def __init__(self, storage, before=None):
|
||||
Base.__init__(self, storage)
|
||||
self._before = before
|
||||
|
||||
def isReadOnly(self):
|
||||
return True
|
||||
|
||||
def supportsUndo(self):
|
||||
return False
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
release = self._storage.release
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
release()
|
||||
|
||||
close = release
|
||||
|
||||
def sync(self, force=True):
|
||||
pass
|
||||
|
||||
def poll_invalidations(self):
|
||||
return []
|
||||
|
||||
new_oid = pack = store = read_only_writer
|
||||
|
||||
def load(self, oid, version=''):
|
||||
r = self._storage.loadBefore(oid, self._before)
|
||||
if r is None:
|
||||
raise POSException.POSKeyError(oid)
|
||||
return r[:2]
|
||||
|
||||
|
||||
class UndoAdapterInstance(Base):
|
||||
|
||||
_copy_methods = Base._copy_methods + (
|
||||
'tpc_abort',
|
||||
)
|
||||
|
||||
def __init__(self, base):
|
||||
self._base = base
|
||||
Base.__init__(self, base._storage)
|
||||
|
||||
def release(self):
|
||||
pass
|
||||
|
||||
close = release
|
||||
|
||||
def tpc_begin(self, transaction):
|
||||
self._storage.tpc_begin(transaction)
|
||||
self._undone = set()
|
||||
|
||||
def undo(self, transaction_id, transaction):
|
||||
result = self._storage.undo(transaction_id, transaction)
|
||||
if result:
|
||||
self._undone.update(result[1])
|
||||
return result
|
||||
|
||||
def tpc_vote(self, transaction):
|
||||
result = self._storage.tpc_vote(transaction)
|
||||
if result:
|
||||
self._undone.update(result)
|
||||
|
||||
def tpc_finish(self, transaction, func = lambda tid: None):
|
||||
|
||||
def invalidate_finish(tid):
|
||||
self._base._invalidate_finish(self._undone, None)
|
||||
func(tid)
|
||||
|
||||
self._storage.tpc_finish(transaction, invalidate_finish)
|
||||
@@ -0,0 +1,227 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Persistent Class Support
|
||||
|
||||
$Id$
|
||||
"""
|
||||
|
||||
|
||||
# Notes:
|
||||
#
|
||||
# Persistent classes are non-ghostable. This has some interesting
|
||||
# ramifications:
|
||||
#
|
||||
# - When an object is invalidated, it must reload its state
|
||||
#
|
||||
# - When an object is loaded from the database, its state must be
|
||||
# loaded. Unfortunately, there isn't a clear signal when an object is
|
||||
# loaded from the database. This should probably be fixed.
|
||||
#
|
||||
# In the mean time, we need to infer. This should be viewed as a
|
||||
# short term hack.
|
||||
#
|
||||
# Here's the strategy we'll use:
|
||||
#
|
||||
# - We'll have a need to be loaded flag that we'll set in
|
||||
# __new__, through an extra argument.
|
||||
#
|
||||
# - When setting _p_oid and _p_jar, if both are set and we need to be
|
||||
# loaded, then we'll load out state.
|
||||
#
|
||||
# - We'll use _p_changed is None to indicate that we're in this state.
|
||||
#
|
||||
|
||||
class _p_DataDescr(object):
|
||||
# Descr used as base for _p_ data. Data are stored in
|
||||
# _p_class_dict.
|
||||
|
||||
def __init__(self, name):
|
||||
self.__name__ = name
|
||||
|
||||
def __get__(self, inst, cls):
|
||||
if inst is None:
|
||||
return self
|
||||
|
||||
if '__global_persistent_class_not_stored_in_DB__' in inst.__dict__:
|
||||
raise AttributeError(self.__name__)
|
||||
return inst._p_class_dict.get(self.__name__)
|
||||
|
||||
def __set__(self, inst, v):
|
||||
inst._p_class_dict[self.__name__] = v
|
||||
|
||||
def __delete__(self, inst):
|
||||
raise AttributeError(self.__name__)
|
||||
|
||||
class _p_oid_or_jar_Descr(_p_DataDescr):
|
||||
# Special descr for _p_oid and _p_jar that loads
|
||||
# state when set if both are set and _p_changed is None
|
||||
#
|
||||
# See notes above
|
||||
|
||||
def __set__(self, inst, v):
|
||||
get = inst._p_class_dict.get
|
||||
if v == get(self.__name__):
|
||||
return
|
||||
|
||||
inst._p_class_dict[self.__name__] = v
|
||||
|
||||
jar = get('_p_jar')
|
||||
if (jar is not None
|
||||
and get('_p_oid') is not None
|
||||
and get('_p_changed') is None
|
||||
):
|
||||
jar.setstate(inst)
|
||||
|
||||
class _p_ChangedDescr(object):
|
||||
# descriptor to handle special weird semantics of _p_changed
|
||||
|
||||
def __get__(self, inst, cls):
|
||||
if inst is None:
|
||||
return self
|
||||
return inst._p_class_dict['_p_changed']
|
||||
|
||||
def __set__(self, inst, v):
|
||||
if v is None:
|
||||
return
|
||||
inst._p_class_dict['_p_changed'] = bool(v)
|
||||
|
||||
def __delete__(self, inst):
|
||||
inst._p_invalidate()
|
||||
|
||||
class _p_MethodDescr(object):
|
||||
"""Provide unassignable class attributes
|
||||
"""
|
||||
|
||||
def __init__(self, func):
|
||||
self.func = func
|
||||
|
||||
def __get__(self, inst, cls):
|
||||
if inst is None:
|
||||
return cls
|
||||
return self.func.__get__(inst, cls)
|
||||
|
||||
def __set__(self, inst, v):
|
||||
raise AttributeError(self.__name__)
|
||||
|
||||
def __delete__(self, inst):
|
||||
raise AttributeError(self.__name__)
|
||||
|
||||
|
||||
special_class_descrs = '__dict__', '__weakref__'
|
||||
|
||||
class PersistentMetaClass(type):
|
||||
|
||||
_p_jar = _p_oid_or_jar_Descr('_p_jar')
|
||||
_p_oid = _p_oid_or_jar_Descr('_p_oid')
|
||||
_p_changed = _p_ChangedDescr()
|
||||
_p_serial = _p_DataDescr('_p_serial')
|
||||
|
||||
def __new__(self, name, bases, cdict, _p_changed=False):
|
||||
cdict = dict([(k, v) for (k, v) in cdict.items()
|
||||
if not k.startswith('_p_')])
|
||||
cdict['_p_class_dict'] = {'_p_changed': _p_changed}
|
||||
return super(PersistentMetaClass, self).__new__(
|
||||
self, name, bases, cdict)
|
||||
|
||||
def __getnewargs__(self):
|
||||
return self.__name__, self.__bases__, {}, None
|
||||
|
||||
__getnewargs__ = _p_MethodDescr(__getnewargs__)
|
||||
|
||||
def _p_maybeupdate(self, name):
|
||||
get = self._p_class_dict.get
|
||||
data_manager = get('_p_jar')
|
||||
|
||||
if (
|
||||
(data_manager is not None)
|
||||
and
|
||||
(get('_p_oid') is not None)
|
||||
and
|
||||
(get('_p_changed') == False)
|
||||
):
|
||||
|
||||
self._p_changed = True
|
||||
data_manager.register(self)
|
||||
|
||||
def __setattr__(self, name, v):
|
||||
if not ((name.startswith('_p_') or name.startswith('_v'))):
|
||||
self._p_maybeupdate(name)
|
||||
super(PersistentMetaClass, self).__setattr__(name, v)
|
||||
|
||||
def __delattr__(self, name):
|
||||
if not ((name.startswith('_p_') or name.startswith('_v'))):
|
||||
self._p_maybeupdate(name)
|
||||
super(PersistentMetaClass, self).__delattr__(name)
|
||||
|
||||
def _p_deactivate(self):
|
||||
# persistent classes can't be ghosts
|
||||
pass
|
||||
|
||||
_p_deactivate = _p_MethodDescr(_p_deactivate)
|
||||
|
||||
def _p_invalidate(self):
|
||||
# reset state
|
||||
self._p_class_dict['_p_changed'] = None
|
||||
self._p_jar.setstate(self)
|
||||
|
||||
_p_invalidate = _p_MethodDescr(_p_invalidate)
|
||||
|
||||
|
||||
def __getstate__(self):
|
||||
return (self.__bases__,
|
||||
dict([(k, v) for (k, v) in self.__dict__.items()
|
||||
if not (k.startswith('_p_')
|
||||
or k.startswith('_v_')
|
||||
or k in special_class_descrs
|
||||
)
|
||||
]),
|
||||
)
|
||||
|
||||
__getstate__ = _p_MethodDescr(__getstate__)
|
||||
|
||||
def __setstate__(self, state):
|
||||
bases, cdict = state
|
||||
if self.__bases__ != bases:
|
||||
# __getnewargs__ should've taken care of that
|
||||
raise AssertionError(self.__bases__, '!=', bases)
|
||||
cdict = dict([(k, v) for (k, v) in cdict.items()
|
||||
if not k.startswith('_p_')])
|
||||
|
||||
_p_class_dict = self._p_class_dict
|
||||
self._p_class_dict = {}
|
||||
|
||||
to_remove = [k for k in self.__dict__
|
||||
if ((k not in cdict)
|
||||
and
|
||||
(k not in special_class_descrs)
|
||||
and
|
||||
(k != '_p_class_dict')
|
||||
)]
|
||||
|
||||
for k in to_remove:
|
||||
delattr(self, k)
|
||||
|
||||
for k, v in cdict.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
self._p_class_dict = _p_class_dict
|
||||
|
||||
self._p_changed = False
|
||||
|
||||
__setstate__ = _p_MethodDescr(__setstate__)
|
||||
|
||||
def _p_activate(self):
|
||||
self._p_jar.setstate(self)
|
||||
|
||||
_p_activate = _p_MethodDescr(_p_activate)
|
||||
@@ -0,0 +1,291 @@
|
||||
==================
|
||||
Persistent Classes
|
||||
==================
|
||||
|
||||
NOTE: persistent classes are EXPERIMENTAL and, in some sense,
|
||||
incomplete. This module exists largely to test changes made to
|
||||
support Zope 2 ZClasses, with their historical flaws.
|
||||
|
||||
The persistentclass module provides a meta class that can be used to implement
|
||||
persistent classes.
|
||||
|
||||
Persistent classes have the following properties:
|
||||
|
||||
- They cannot be turned into ghosts
|
||||
|
||||
- They can only contain picklable subobjects
|
||||
|
||||
- They don't live in regular file-system modules
|
||||
|
||||
Let's look at an example:
|
||||
|
||||
>>> def __init__(self, name):
|
||||
... self.name = name
|
||||
|
||||
>>> def foo(self):
|
||||
... return self.name, self.kind
|
||||
|
||||
>>> import ZODB.persistentclass
|
||||
>>> C = ZODB.persistentclass.PersistentMetaClass(
|
||||
... 'C', (object, ), dict(
|
||||
... __init__ = __init__,
|
||||
... __module__ = '__zodb__',
|
||||
... foo = foo,
|
||||
... kind = 'sample',
|
||||
... ))
|
||||
|
||||
This example is obviously a bit contrived. In particular, we defined
|
||||
the methods outside of the class. Why? Because all of the items in a
|
||||
persistent class must be picklable. We defined the methods as global
|
||||
functions to make them picklable.
|
||||
|
||||
Also note that we explicitly set the module. Persistent classes don't
|
||||
live in normal Python modules. Rather, they live in the database. We
|
||||
use information in ``__module__`` to record where in the database. When
|
||||
we want to use a database, we will need to supply a custom class
|
||||
factory to load instances of the class.
|
||||
|
||||
The class we created works a lot like other persistent objects. It
|
||||
has standard standard persistent attributes:
|
||||
|
||||
>>> C._p_oid
|
||||
>>> C._p_jar
|
||||
>>> C._p_serial
|
||||
>>> C._p_changed
|
||||
False
|
||||
|
||||
Because we haven't saved the object, the jar, oid, and serial are all
|
||||
None and it's not changed.
|
||||
|
||||
We can create and use instances of the class:
|
||||
|
||||
>>> c = C('first')
|
||||
>>> c.foo()
|
||||
('first', 'sample')
|
||||
|
||||
We can modify the class and none of the persistent attributes will
|
||||
change because the object hasn't been saved.
|
||||
|
||||
>>> import six
|
||||
>>> def bar(self):
|
||||
... six.print_('bar', self.name)
|
||||
>>> C.bar = bar
|
||||
>>> c.bar()
|
||||
bar first
|
||||
|
||||
>>> C._p_oid
|
||||
>>> C._p_jar
|
||||
>>> C._p_serial
|
||||
>>> C._p_changed
|
||||
False
|
||||
|
||||
Now, we can store the class in a database. We're going to use an
|
||||
explicit transaction manager so that we can show parallel transactions
|
||||
without having to use threads.
|
||||
|
||||
>>> import transaction
|
||||
>>> tm = transaction.TransactionManager()
|
||||
>>> connection = some_database.open(transaction_manager=tm)
|
||||
>>> connection.root()['C'] = C
|
||||
>>> tm.commit()
|
||||
|
||||
Now, if we look at the persistence variables, we'll see that they have
|
||||
values:
|
||||
|
||||
>>> C._p_oid
|
||||
'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
>>> C._p_jar is not None
|
||||
True
|
||||
>>> C._p_serial is not None
|
||||
True
|
||||
>>> C._p_changed
|
||||
False
|
||||
|
||||
Now, if we modify the class:
|
||||
|
||||
>>> def baz(self):
|
||||
... six.print_('baz', self.name)
|
||||
>>> C.baz = baz
|
||||
>>> c.baz()
|
||||
baz first
|
||||
|
||||
We'll see that the class has changed:
|
||||
|
||||
>>> C._p_changed
|
||||
True
|
||||
|
||||
If we abort the transaction:
|
||||
|
||||
>>> tm.abort()
|
||||
|
||||
Then the class will return to it's prior state:
|
||||
|
||||
>>> c.baz()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
AttributeError: 'C' object has no attribute 'baz'
|
||||
|
||||
>>> c.bar()
|
||||
bar first
|
||||
|
||||
We can open another connection and access the class there.
|
||||
|
||||
>>> tm2 = transaction.TransactionManager()
|
||||
>>> connection2 = some_database.open(transaction_manager=tm2)
|
||||
|
||||
>>> C2 = connection2.root()['C']
|
||||
>>> c2 = C2('other')
|
||||
>>> c2.bar()
|
||||
bar other
|
||||
|
||||
If we make changes without committing them:
|
||||
|
||||
>>> C.bar = baz
|
||||
>>> c.bar()
|
||||
baz first
|
||||
|
||||
>>> C is C2
|
||||
False
|
||||
|
||||
Other connections are unaffected:
|
||||
|
||||
>>> connection2.sync()
|
||||
>>> c2.bar()
|
||||
bar other
|
||||
|
||||
Until we commit:
|
||||
|
||||
>>> tm.commit()
|
||||
>>> connection2.sync()
|
||||
>>> c2.bar()
|
||||
baz other
|
||||
|
||||
Similarly, we don't see changes made in other connections:
|
||||
|
||||
>>> C2.color = 'red'
|
||||
>>> tm2.commit()
|
||||
|
||||
>>> c.color
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
AttributeError: 'C' object has no attribute 'color'
|
||||
|
||||
until we sync:
|
||||
|
||||
>>> connection.sync()
|
||||
>>> c.color
|
||||
'red'
|
||||
|
||||
Instances of Persistent Classes
|
||||
-------------------------------
|
||||
|
||||
We can, of course, store instances of persistent classes in the
|
||||
database:
|
||||
|
||||
>>> c.color = 'blue'
|
||||
>>> connection.root()['c'] = c
|
||||
>>> tm.commit()
|
||||
|
||||
>>> connection2.sync()
|
||||
>>> connection2.root()['c'].color
|
||||
'blue'
|
||||
|
||||
NOTE: If a non-persistent instance of a persistent class is copied,
|
||||
the class may be copied as well. This is usually not the desired
|
||||
result.
|
||||
|
||||
|
||||
Persistent instances of persistent classes
|
||||
------------------------------------------
|
||||
|
||||
Persistent instances of persistent classes are handled differently
|
||||
than normal instances. When we copy a persistent instances of a
|
||||
persistent class, we want to avoid copying the class.
|
||||
|
||||
Lets create a persistent class that subclasses Persistent:
|
||||
|
||||
>>> import persistent
|
||||
>>> class P(persistent.Persistent, C):
|
||||
... __module__ = '__zodb__'
|
||||
... color = 'green'
|
||||
|
||||
>>> connection.root()['P'] = P
|
||||
|
||||
>>> import persistent.mapping
|
||||
>>> connection.root()['obs'] = persistent.mapping.PersistentMapping()
|
||||
>>> p = P('p')
|
||||
>>> connection.root()['obs']['p'] = p
|
||||
>>> tm.commit()
|
||||
|
||||
You might be wondering why we didn't just stick 'p' into the root
|
||||
object. We created an intermediate persistent object instead. We are
|
||||
storing persistent classes in the root object. To create a ghost for a
|
||||
persistent instance of a persistent class, we need to be able to be
|
||||
able to access the root object and it must be loaded first. If the
|
||||
instance was in the root object, we'd be unable to create it while
|
||||
loading the root object.
|
||||
|
||||
Now, if we try to load it, we get a broken object:
|
||||
|
||||
>>> connection2.sync()
|
||||
>>> connection2.root()['obs']['p']
|
||||
<persistent broken __zodb__.P instance '\x00\x00\x00\x00\x00\x00\x00\x04'>
|
||||
|
||||
because the module, `__zodb__` can't be loaded. We need to provide a
|
||||
class factory that knows about this special module. Here we'll supply a
|
||||
sample class factory that looks up a class name in the database root
|
||||
if the module is `__zodb__`. It falls back to the normal class lookup
|
||||
for other modules:
|
||||
|
||||
>>> from ZODB.broken import find_global
|
||||
>>> def classFactory(connection, modulename, globalname):
|
||||
... if modulename == '__zodb__':
|
||||
... return connection.root()[globalname]
|
||||
... return find_global(modulename, globalname)
|
||||
|
||||
>>> some_database.classFactory = classFactory
|
||||
|
||||
Normally, the classFactory should be set before a database is opened.
|
||||
We'll reopen the connections we're using. We'll assign the old
|
||||
connections to a variable first to prevent getting them from the
|
||||
connection pool:
|
||||
|
||||
>>> old = connection, connection2
|
||||
>>> connection = some_database.open(transaction_manager=tm)
|
||||
>>> connection2 = some_database.open(transaction_manager=tm2)
|
||||
|
||||
Now, we can read the object:
|
||||
|
||||
>>> connection2.root()['obs']['p'].color
|
||||
'green'
|
||||
>>> connection2.root()['obs']['p'].color = 'blue'
|
||||
>>> tm2.commit()
|
||||
|
||||
>>> connection.sync()
|
||||
>>> p = connection.root()['obs']['p']
|
||||
>>> p.color
|
||||
'blue'
|
||||
|
||||
Copying
|
||||
-------
|
||||
|
||||
If we copy an instance via export/import, the copy and the original
|
||||
share the same class:
|
||||
|
||||
>>> file = connection.exportFile(p._p_oid)
|
||||
>>> _ = file.seek(0)
|
||||
>>> cp = connection.importFile(file)
|
||||
>>> file.close()
|
||||
>>> cp.color
|
||||
'blue'
|
||||
|
||||
>>> cp is not p
|
||||
True
|
||||
|
||||
>>> cp.__class__ is p.__class__
|
||||
True
|
||||
|
||||
>>> tm.abort()
|
||||
|
||||
|
||||
XXX test abort of import
|
||||
@@ -0,0 +1,107 @@
|
||||
This directory contains a collection of utilities for managing ZODB
|
||||
databases. Some are more useful than others. If you install ZODB
|
||||
using distutils ("python setup.py install"), a few of these will be installed.
|
||||
|
||||
Unless otherwise noted, these scripts are invoked with the name of the
|
||||
Data.fs file as their only argument. Example: checkbtrees.py data.fs.
|
||||
|
||||
|
||||
analyze.py -- a transaction analyzer for FileStorage
|
||||
|
||||
Reports on the data in a FileStorage. The report is organized by
|
||||
class. It shows total data, as well as separate reports for current
|
||||
and historical revisions of objects.
|
||||
|
||||
|
||||
checkbtrees.py -- checks BTrees in a FileStorage for corruption
|
||||
|
||||
Attempts to find all the BTrees contained in a Data.fs, calls their
|
||||
_check() methods, and runs them through BTrees.check.check().
|
||||
|
||||
|
||||
fsdump.py -- summarize FileStorage contents, one line per revision
|
||||
|
||||
Prints a report of FileStorage contents, with one line for each
|
||||
transaction and one line for each data record in that transaction.
|
||||
Includes time stamps, file positions, and class names.
|
||||
|
||||
|
||||
fsoids.py -- trace all uses of specified oids in a FileStorage
|
||||
|
||||
For heavy debugging.
|
||||
A set of oids is specified by text file listing and/or command line.
|
||||
A report is generated showing all uses of these oids in the database:
|
||||
all new-revision creation/modifications, all references from all
|
||||
revisions of other objects, and all creation undos.
|
||||
|
||||
|
||||
fstest.py -- simple consistency checker for FileStorage
|
||||
|
||||
usage: fstest.py [-v] data.fs
|
||||
|
||||
The fstest tool will scan all the data in a FileStorage and report an
|
||||
error if it finds any corrupt transaction data. The tool will print a
|
||||
message when the first error is detected an exit.
|
||||
|
||||
The tool accepts one or more -v arguments. If a single -v is used, it
|
||||
will print a line of text for each transaction record it encounters.
|
||||
If two -v arguments are used, it will also print a line of text for
|
||||
each object. The objects for a transaction will be printed before the
|
||||
transaction itself.
|
||||
|
||||
Note: It does not check the consistency of the object pickles. It is
|
||||
possible for the damage to occur only in the part of the file that
|
||||
stores object pickles. Those errors will go undetected.
|
||||
|
||||
|
||||
space.py -- report space used by objects in a FileStorage
|
||||
|
||||
usage: space.py [-v] data.fs
|
||||
|
||||
This ignores revisions and versions.
|
||||
|
||||
|
||||
netspace.py -- hackish attempt to report on size of objects
|
||||
|
||||
usage: netspace.py [-P | -v] data.fs
|
||||
|
||||
-P: do a pack first
|
||||
-v: print info for all objects, even if a traversal path isn't found
|
||||
|
||||
Traverses objects from the database root and attempts to calculate
|
||||
size of object, including all reachable subobjects.
|
||||
|
||||
|
||||
repozo.py -- incremental backup utility for FileStorage
|
||||
|
||||
Run the script with the -h option to see usage details.
|
||||
|
||||
|
||||
timeout.py -- script to test transaction timeout
|
||||
|
||||
usage: timeout.py address delay [storage-name]
|
||||
|
||||
This script connects to a storage, begins a transaction, calls store()
|
||||
and tpc_vote(), and then sleeps forever. This should trigger the
|
||||
transaction timeout feature of the server.
|
||||
|
||||
zodbload.py -- exercise ZODB under a heavy synthesized Zope-like load
|
||||
|
||||
See the module docstring for details. Note that this script requires
|
||||
Zope. New in ZODB3 3.1.4.
|
||||
|
||||
|
||||
fsrefs.py -- check FileStorage for dangling references
|
||||
|
||||
|
||||
fstail.py -- display the most recent transactions in a FileStorage
|
||||
|
||||
usage: fstail.py [-n nxtn] data.fs
|
||||
|
||||
The most recent ntxn transactions are displayed, to stdout.
|
||||
Optional argument -n specifies ntxn, and defaults to 10.
|
||||
|
||||
|
||||
migrate.py -- do a storage migration and gather statistics
|
||||
|
||||
See the module docstring for details.
|
||||
@@ -0,0 +1 @@
|
||||
#
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Based on a transaction analyzer by Matt Kromer.
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from ZODB._compat import PersistentUnpickler, BytesIO
|
||||
|
||||
|
||||
|
||||
class FakeError(Exception):
|
||||
def __init__(self, module, name):
|
||||
Exception.__init__(self)
|
||||
self.module = module
|
||||
self.name = name
|
||||
|
||||
|
||||
def fake_find_class(module, name):
|
||||
raise FakeError(module, name)
|
||||
|
||||
|
||||
def FakeUnpickler(f):
|
||||
unpickler = PersistentUnpickler(fake_find_class, None, f)
|
||||
return unpickler
|
||||
|
||||
|
||||
class Report(object):
|
||||
def __init__(self):
|
||||
self.OIDMAP = {}
|
||||
self.TYPEMAP = {}
|
||||
self.TYPESIZE = {}
|
||||
self.FREEMAP = {}
|
||||
self.USEDMAP = {}
|
||||
self.TIDS = 0
|
||||
self.OIDS = 0
|
||||
self.DBYTES = 0
|
||||
self.COIDS = 0
|
||||
self.CBYTES = 0
|
||||
self.FOIDS = 0
|
||||
self.FBYTES = 0
|
||||
|
||||
def shorten(s, n):
|
||||
l = len(s)
|
||||
if l <= n:
|
||||
return s
|
||||
while len(s) + 3 > n: # account for ...
|
||||
i = s.find(".")
|
||||
if i == -1:
|
||||
# In the worst case, just return the rightmost n bytes
|
||||
return s[-n:]
|
||||
else:
|
||||
s = s[i + 1:]
|
||||
l = len(s)
|
||||
return "..." + s
|
||||
|
||||
def report(rep):
|
||||
print("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
|
||||
print("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
|
||||
print(("Average transaction size is %7.2f bytes" %
|
||||
(rep.DBYTES * 1.0 / rep.TIDS)))
|
||||
|
||||
print("Types used:")
|
||||
fmt = "%-46s %7s %9s %6s %7s"
|
||||
fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
|
||||
fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
|
||||
print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize"))
|
||||
print(fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7))
|
||||
typemap = sorted(rep.TYPEMAP)
|
||||
cumpct = 0.0
|
||||
for t in typemap:
|
||||
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
|
||||
cumpct += pct
|
||||
print(fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t],
|
||||
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
|
||||
|
||||
print(fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
|
||||
print("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
|
||||
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
|
||||
print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
|
||||
rep.DBYTES * 1.0 / rep.OIDS))
|
||||
|
||||
print(fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
|
||||
rep.CBYTES * 100.0 / rep.DBYTES,
|
||||
rep.CBYTES * 1.0 / rep.COIDS))
|
||||
if rep.FOIDS:
|
||||
print(fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
|
||||
rep.FBYTES * 100.0 / rep.DBYTES,
|
||||
rep.FBYTES * 1.0 / rep.FOIDS))
|
||||
|
||||
def analyze(path):
|
||||
fs = FileStorage(path, read_only=1)
|
||||
fsi = fs.iterator()
|
||||
report = Report()
|
||||
for txn in fsi:
|
||||
analyze_trans(report, txn)
|
||||
return report
|
||||
|
||||
def analyze_trans(report, txn):
|
||||
report.TIDS += 1
|
||||
for rec in txn:
|
||||
analyze_rec(report, rec)
|
||||
|
||||
def get_type(record):
|
||||
try:
|
||||
unpickled = FakeUnpickler(BytesIO(record.data)).load()
|
||||
except FakeError as err:
|
||||
return "%s.%s" % (err.module, err.name)
|
||||
classinfo = unpickled[0]
|
||||
if isinstance(classinfo, tuple):
|
||||
mod, klass = classinfo
|
||||
return "%s.%s" % (mod, klass)
|
||||
else:
|
||||
return str(classinfo)
|
||||
|
||||
def analyze_rec(report, record):
|
||||
oid = record.oid
|
||||
report.OIDS += 1
|
||||
if record.data is None:
|
||||
# No pickle -- aborted version or undo of object creation.
|
||||
return
|
||||
try:
|
||||
size = len(record.data) # Ignores various overhead
|
||||
report.DBYTES += size
|
||||
if oid not in report.OIDMAP:
|
||||
type = get_type(record)
|
||||
report.OIDMAP[oid] = type
|
||||
report.USEDMAP[oid] = size
|
||||
report.COIDS += 1
|
||||
report.CBYTES += size
|
||||
else:
|
||||
type = report.OIDMAP[oid]
|
||||
fsize = report.USEDMAP[oid]
|
||||
report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize
|
||||
report.USEDMAP[oid] = size
|
||||
report.FOIDS += 1
|
||||
report.FBYTES += fsize
|
||||
report.CBYTES += size - fsize
|
||||
report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
|
||||
report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
|
||||
except Exception as err:
|
||||
print(err)
|
||||
|
||||
if __name__ == "__main__":
|
||||
path = sys.argv[1]
|
||||
report(analyze(path))
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python
|
||||
"""Check the consistency of BTrees in a Data.fs
|
||||
|
||||
usage: checkbtrees.py data.fs
|
||||
|
||||
Try to find all the BTrees in a Data.fs, call their _check() methods,
|
||||
and run them through BTrees.check.check().
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import ZODB
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from BTrees.check import check
|
||||
|
||||
# Set of oids we've already visited. Since the object structure is
|
||||
# a general graph, this is needed to prevent unbounded paths in the
|
||||
# presence of cycles. It's also helpful in eliminating redundant
|
||||
# checking when a BTree is pointed to by many objects.
|
||||
oids_seen = {}
|
||||
|
||||
# Append (obj, path) to L if and only if obj is a persistent object
|
||||
# and we haven't seen it before.
|
||||
def add_if_new_persistent(L, obj, path):
|
||||
global oids_seen
|
||||
|
||||
getattr(obj, '_', None) # unghostify
|
||||
if hasattr(obj, '_p_oid'):
|
||||
oid = obj._p_oid
|
||||
if oid not in oids_seen:
|
||||
L.append((obj, path))
|
||||
oids_seen[oid] = 1
|
||||
|
||||
def get_subobjects(obj):
|
||||
getattr(obj, '_', None) # unghostify
|
||||
sub = []
|
||||
try:
|
||||
attrs = obj.__dict__.items()
|
||||
except AttributeError:
|
||||
attrs = ()
|
||||
for pair in attrs:
|
||||
sub.append(pair)
|
||||
|
||||
# what if it is a mapping?
|
||||
try:
|
||||
items = obj.items()
|
||||
except AttributeError:
|
||||
items = ()
|
||||
for k, v in items:
|
||||
if not isinstance(k, int):
|
||||
sub.append(("<key>", k))
|
||||
if not isinstance(v, int):
|
||||
sub.append(("[%s]" % repr(k), v))
|
||||
|
||||
# what if it is a sequence?
|
||||
i = 0
|
||||
while 1:
|
||||
try:
|
||||
elt = obj[i]
|
||||
except:
|
||||
break
|
||||
sub.append(("[%d]" % i, elt))
|
||||
i += 1
|
||||
|
||||
return sub
|
||||
|
||||
def main(fname=None):
|
||||
if fname is None:
|
||||
import sys
|
||||
try:
|
||||
fname, = sys.argv[1:]
|
||||
except:
|
||||
print(__doc__)
|
||||
sys.exit(2)
|
||||
|
||||
fs = FileStorage(fname, read_only=1)
|
||||
cn = ZODB.DB(fs).open()
|
||||
rt = cn.root()
|
||||
todo = []
|
||||
add_if_new_persistent(todo, rt, '')
|
||||
|
||||
found = 0
|
||||
while todo:
|
||||
obj, path = todo.pop(0)
|
||||
found += 1
|
||||
if not path:
|
||||
print("<root>", repr(obj))
|
||||
else:
|
||||
print(path, repr(obj))
|
||||
|
||||
mod = str(obj.__class__.__module__)
|
||||
if mod.startswith("BTrees"):
|
||||
if hasattr(obj, "_check"):
|
||||
try:
|
||||
obj._check()
|
||||
except AssertionError as msg:
|
||||
print("*" * 60)
|
||||
print(msg)
|
||||
print("*" * 60)
|
||||
|
||||
try:
|
||||
check(obj)
|
||||
except AssertionError as msg:
|
||||
print("*" * 60)
|
||||
print(msg)
|
||||
print("*" * 60)
|
||||
|
||||
if found % 100 == 0:
|
||||
cn.cacheMinimize()
|
||||
|
||||
for k, v in get_subobjects(obj):
|
||||
if k.startswith('['):
|
||||
# getitem
|
||||
newpath = "%s%s" % (path, k)
|
||||
else:
|
||||
newpath = "%s.%s" % (path, k)
|
||||
add_if_new_persistent(todo, v, newpath)
|
||||
|
||||
print("total", len(fs._index), "found", found)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
"""FileStorage oid-tracer.
|
||||
|
||||
usage: fsoids.py [-f oid_file] Data.fs [oid]...
|
||||
|
||||
Display information about all occurrences of specified oids in a FileStorage.
|
||||
This is meant for heavy debugging.
|
||||
|
||||
This includes all revisions of the oids, all objects referenced by the
|
||||
oids, and all revisions of all objects referring to the oids.
|
||||
|
||||
If specified, oid_file is an input text file, containing one oid per
|
||||
line. oids are specified as integers, in any of Python's integer
|
||||
notations (typically like 0x341a). One or more oids can also be specified
|
||||
on the command line.
|
||||
|
||||
The output is grouped by oid, from smallest to largest, and sub-grouped
|
||||
by transaction, from oldest to newest.
|
||||
|
||||
This will not alter the FileStorage, but running against a live FileStorage
|
||||
is not recommended (spurious error messages may result).
|
||||
|
||||
See testfsoids.py for a tutorial doctest.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
from ZODB.FileStorage.fsoids import Tracer
|
||||
|
||||
def usage():
|
||||
print(__doc__)
|
||||
|
||||
def main():
|
||||
import getopt
|
||||
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'f:')
|
||||
if not args:
|
||||
usage()
|
||||
raise ValueError("Must specify a FileStorage")
|
||||
path = None
|
||||
for k, v in opts:
|
||||
if k == '-f':
|
||||
path = v
|
||||
except (getopt.error, ValueError):
|
||||
usage()
|
||||
raise
|
||||
|
||||
c = Tracer(args[0])
|
||||
for oid in args[1:]:
|
||||
as_int = int(oid, 0) # 0 == auto-detect base
|
||||
c.register_oids(as_int)
|
||||
if path is not None:
|
||||
for line in open(path):
|
||||
as_int = int(line, 0)
|
||||
c.register_oids(as_int)
|
||||
if not c.oids:
|
||||
raise ValueError("no oids specified")
|
||||
c.run()
|
||||
c.report()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
"""Check FileStorage for dangling references.
|
||||
|
||||
usage: fsrefs.py [-v] data.fs
|
||||
|
||||
fsrefs.py checks object sanity by trying to load the current revision of
|
||||
every object O in the database, and also verifies that every object
|
||||
directly reachable from each such O exists in the database.
|
||||
|
||||
It's hard to explain exactly what it does because it relies on undocumented
|
||||
features in Python's cPickle module: many of the crucial steps of loading
|
||||
an object are taken, but application objects aren't actually created. This
|
||||
saves a lot of time, and allows fsrefs to be run even if the code
|
||||
implementing the object classes isn't available.
|
||||
|
||||
A read-only connection to the specified FileStorage is made, but it is not
|
||||
recommended to run fsrefs against a live FileStorage. Because a live
|
||||
FileStorage is mutating while fsrefs runs, it's not possible for fsrefs to
|
||||
get a wholly consistent view of the database across the entire time fsrefs
|
||||
is running; spurious error messages may result.
|
||||
|
||||
fsrefs doesn't normally produce any output. If an object fails to load, the
|
||||
oid of the object is given in a message saying so, and if -v was specified
|
||||
then the traceback corresponding to the load failure is also displayed
|
||||
(this is the only effect of the -v flag).
|
||||
|
||||
Three other kinds of errors are also detected, when an object O loads OK,
|
||||
and directly refers to a persistent object P but there's a problem with P:
|
||||
|
||||
- If P doesn't exist in the database, a message saying so is displayed.
|
||||
The unsatisifiable reference to P is often called a "dangling
|
||||
reference"; P is called "missing" in the error output.
|
||||
|
||||
- If the current state of the database is such that P's creation has
|
||||
been undone, then P can't be loaded either. This is also a kind of
|
||||
dangling reference, but is identified as "object creation was undone".
|
||||
|
||||
- If P can't be loaded (but does exist in the database), a message saying
|
||||
that O refers to an object that can't be loaded is displayed.
|
||||
|
||||
fsrefs also (indirectly) checks that the .index file is sane, because
|
||||
fsrefs uses the index to get its idea of what constitutes "all the objects
|
||||
in the database".
|
||||
|
||||
Note these limitations: because fsrefs only looks at the current revision
|
||||
of objects, it does not attempt to load objects in versions, or non-current
|
||||
revisions of objects; therefore fsrefs cannot find problems in versions or
|
||||
in non-current revisions.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import traceback
|
||||
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from ZODB.TimeStamp import TimeStamp
|
||||
from ZODB.utils import u64, oid_repr, get_pickle_metadata, load_current
|
||||
from ZODB.serialize import get_refs
|
||||
from ZODB.POSException import POSKeyError
|
||||
|
||||
# There's a problem with oid. 'data' is its pickle, and 'serial' its
|
||||
# serial number. 'missing' is a list of (oid, class, reason) triples,
|
||||
# explaining what the problem(s) is(are).
|
||||
def report(oid, data, serial, missing):
|
||||
from_mod, from_class = get_pickle_metadata(data)
|
||||
if len(missing) > 1:
|
||||
plural = "s"
|
||||
else:
|
||||
plural = ""
|
||||
ts = TimeStamp(serial)
|
||||
print("oid %s %s.%s" % (hex(u64(oid)), from_mod, from_class))
|
||||
print("last updated: %s, tid=%s" % (ts, hex(u64(serial))))
|
||||
print("refers to invalid object%s:" % plural)
|
||||
for oid, info, reason in missing:
|
||||
if isinstance(info, tuple):
|
||||
description = "%s.%s" % info
|
||||
else:
|
||||
description = str(info)
|
||||
print("\toid %s %s: %r" % (oid_repr(oid), reason, description))
|
||||
print()
|
||||
|
||||
def main(path=None):
|
||||
verbose = 0
|
||||
if path is None:
|
||||
import sys
|
||||
import getopt
|
||||
|
||||
opts, args = getopt.getopt(sys.argv[1:], "v")
|
||||
for k, v in opts:
|
||||
if k == "-v":
|
||||
verbose += 1
|
||||
|
||||
path, = args
|
||||
|
||||
|
||||
fs = FileStorage(path, read_only=1)
|
||||
|
||||
# Set of oids in the index that failed to load due to POSKeyError.
|
||||
# This is what happens if undo is applied to the transaction creating
|
||||
# the object (the oid is still in the index, but its current data
|
||||
# record has a backpointer of 0, and POSKeyError is raised then
|
||||
# because of that backpointer).
|
||||
undone = {}
|
||||
|
||||
# Set of oids that were present in the index but failed to load.
|
||||
# This does not include oids in undone.
|
||||
noload = {}
|
||||
|
||||
for oid in fs._index.keys():
|
||||
try:
|
||||
data, serial = load_current(fs, oid)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except POSKeyError:
|
||||
undone[oid] = 1
|
||||
except:
|
||||
if verbose:
|
||||
traceback.print_exc()
|
||||
noload[oid] = 1
|
||||
|
||||
inactive = noload.copy()
|
||||
inactive.update(undone)
|
||||
for oid in fs._index.keys():
|
||||
if oid in inactive:
|
||||
continue
|
||||
data, serial = load_current(fs, oid)
|
||||
refs = get_refs(data)
|
||||
missing = [] # contains 3-tuples of oid, klass-metadata, reason
|
||||
for ref, klass in refs:
|
||||
if klass is None:
|
||||
klass = '<unknown>'
|
||||
if ref not in fs._index:
|
||||
missing.append((ref, klass, "missing"))
|
||||
if ref in noload:
|
||||
missing.append((ref, klass, "failed to load"))
|
||||
if ref in undone:
|
||||
missing.append((ref, klass, "object creation was undone"))
|
||||
if missing:
|
||||
report(oid, data, serial, missing)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python2
|
||||
"""Print details statistics from fsdump output."""
|
||||
from __future__ import print_function
|
||||
import re
|
||||
import sys
|
||||
import six
|
||||
from six.moves import filter
|
||||
|
||||
rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
|
||||
rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
|
||||
|
||||
def sort_byhsize(seq, reverse=False):
|
||||
L = [(v.size(), k, v) for k, v in seq]
|
||||
L.sort()
|
||||
if reverse:
|
||||
L.reverse()
|
||||
return [(k, v) for n, k, v in L]
|
||||
|
||||
class Histogram(dict):
|
||||
|
||||
def add(self, size):
|
||||
self[size] = self.get(size, 0) + 1
|
||||
|
||||
def size(self):
|
||||
return sum(six.itervalues(self))
|
||||
|
||||
def mean(self):
|
||||
product = sum([k * v for k, v in six.iteritems(self)])
|
||||
return product / self.size()
|
||||
|
||||
def median(self):
|
||||
# close enough?
|
||||
n = self.size() / 2
|
||||
L = self.keys()
|
||||
L.sort()
|
||||
L.reverse()
|
||||
while 1:
|
||||
k = L.pop()
|
||||
if self[k] > n:
|
||||
return k
|
||||
n -= self[k]
|
||||
|
||||
def mode(self):
|
||||
mode = 0
|
||||
value = 0
|
||||
for k, v in six.iteritems(self):
|
||||
if v > value:
|
||||
value = v
|
||||
mode = k
|
||||
return mode
|
||||
|
||||
def make_bins(self, binsize):
|
||||
maxkey = max(six.iterkeys(self))
|
||||
self.binsize = binsize
|
||||
self.bins = [0] * (1 + maxkey / binsize)
|
||||
for k, v in six.iteritems(self):
|
||||
b = k / binsize
|
||||
self.bins[b] += v
|
||||
|
||||
def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
|
||||
if usebins:
|
||||
# Use existing bins with whatever size they have
|
||||
binsize = self.binsize
|
||||
else:
|
||||
# Make new bins
|
||||
self.make_bins(binsize)
|
||||
maxval = max(self.bins)
|
||||
# Print up to 40 dots for a value
|
||||
dot = max(maxval / 40, 1)
|
||||
tot = sum(self.bins)
|
||||
print(name)
|
||||
print("Total", tot, end=' ')
|
||||
print("Median", self.median(), end=' ')
|
||||
print("Mean", self.mean(), end=' ')
|
||||
print("Mode", self.mode(), end=' ')
|
||||
print("Max", max(self))
|
||||
print("One * represents", dot)
|
||||
gap = False
|
||||
cum = 0
|
||||
for i, n in enumerate(self.bins):
|
||||
if gaps and (not n or (skip and not n / dot)):
|
||||
if not gap:
|
||||
print(" ...")
|
||||
gap = True
|
||||
continue
|
||||
gap = False
|
||||
p = 100 * n / tot
|
||||
cum += n
|
||||
pc = 100 * cum / tot
|
||||
print("%6d %6d %3d%% %3d%% %s" % (
|
||||
i * binsize, n, p, pc, "*" * (n / dot)))
|
||||
print()
|
||||
|
||||
def class_detail(class_size):
|
||||
# summary of classes
|
||||
fmt = "%5s %6s %6s %6s %-50.50s"
|
||||
labels = ["num", "median", "mean", "mode", "class"]
|
||||
print(fmt % tuple(labels))
|
||||
print(fmt % tuple(["-" * len(s) for s in labels]))
|
||||
for klass, h in sort_byhsize(six.iteritems(class_size)):
|
||||
print(fmt % (h.size(), h.median(), h.mean(), h.mode(), klass))
|
||||
print()
|
||||
|
||||
# per class details
|
||||
for klass, h in sort_byhsize(six.iteritems(class_size), reverse=True):
|
||||
h.make_bins(50)
|
||||
if len(filter(None, h.bins)) == 1:
|
||||
continue
|
||||
h.report("Object size for %s" % klass, usebins=True)
|
||||
|
||||
def revision_detail(lifetimes, classes):
|
||||
# Report per-class details for any object modified more than once
|
||||
for name, oids in six.iteritems(classes):
|
||||
h = Histogram()
|
||||
keep = False
|
||||
for oid in dict.fromkeys(oids, 1):
|
||||
L = lifetimes.get(oid)
|
||||
n = len(L)
|
||||
h.add(n)
|
||||
if n > 1:
|
||||
keep = True
|
||||
if keep:
|
||||
h.report("Number of revisions for %s" % name, binsize=10)
|
||||
|
||||
def main(path=None):
|
||||
if path is None:
|
||||
path = sys.argv[1]
|
||||
txn_objects = Histogram() # histogram of txn size in objects
|
||||
txn_bytes = Histogram() # histogram of txn size in bytes
|
||||
obj_size = Histogram() # histogram of object size
|
||||
n_updates = Histogram() # oid -> num updates
|
||||
n_classes = Histogram() # class -> num objects
|
||||
lifetimes = {} # oid -> list of tids
|
||||
class_size = {} # class -> histogram of object size
|
||||
classes = {} # class -> list of oids
|
||||
|
||||
MAX = 0
|
||||
objects = 0
|
||||
tid = None
|
||||
|
||||
f = open(path, "rb")
|
||||
for i, line in enumerate(f):
|
||||
if MAX and i > MAX:
|
||||
break
|
||||
if line.startswith(" data"):
|
||||
m = rx_data.search(line)
|
||||
if not m:
|
||||
continue
|
||||
oid, klass, size = m.groups()
|
||||
size = int(size)
|
||||
|
||||
obj_size.add(size)
|
||||
n_updates.add(oid)
|
||||
n_classes.add(klass)
|
||||
|
||||
h = class_size.get(klass)
|
||||
if h is None:
|
||||
h = class_size[klass] = Histogram()
|
||||
h.add(size)
|
||||
|
||||
L = lifetimes.setdefault(oid, [])
|
||||
L.append(tid)
|
||||
|
||||
L = classes.setdefault(klass, [])
|
||||
L.append(oid)
|
||||
objects += 1
|
||||
|
||||
elif line.startswith("Trans"):
|
||||
|
||||
if tid is not None:
|
||||
txn_objects.add(objects)
|
||||
|
||||
m = rx_txn.search(line)
|
||||
if not m:
|
||||
continue
|
||||
tid, size = m.groups()
|
||||
size = int(size)
|
||||
objects = 0
|
||||
|
||||
txn_bytes.add(size)
|
||||
f.close()
|
||||
|
||||
print("Summary: %d txns, %d objects, %d revisions" % (
|
||||
txn_objects.size(), len(n_updates), n_updates.size()))
|
||||
print()
|
||||
|
||||
txn_bytes.report("Transaction size (bytes)", binsize=1024)
|
||||
txn_objects.report("Transaction size (objects)", binsize=10)
|
||||
obj_size.report("Object size", binsize=128)
|
||||
|
||||
# object lifetime info
|
||||
h = Histogram()
|
||||
for k, v in lifetimes.items():
|
||||
h.add(len(v))
|
||||
h.report("Number of revisions", binsize=10, skip=False)
|
||||
|
||||
# details about revisions
|
||||
revision_detail(lifetimes, classes)
|
||||
|
||||
class_detail(class_size)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Tool to dump the last few transactions from a FileStorage."""
|
||||
from __future__ import print_function
|
||||
from ZODB.fstools import prev_txn
|
||||
|
||||
import binascii
|
||||
import getopt
|
||||
import sys
|
||||
|
||||
try:
|
||||
from hashlib import sha1
|
||||
except ImportError:
|
||||
from sha import sha as sha1
|
||||
|
||||
def main(path, ntxn):
|
||||
with open(path, "rb") as f:
|
||||
f.seek(0, 2)
|
||||
th = prev_txn(f)
|
||||
i = ntxn
|
||||
while th and i > 0:
|
||||
hash = sha1(th.get_raw_data()).digest()
|
||||
l = len(str(th.get_timestamp())) + 1
|
||||
th.read_meta()
|
||||
print("%s: hash=%s" % (th.get_timestamp(),
|
||||
binascii.hexlify(hash).decode()))
|
||||
print(("user=%r description=%r length=%d offset=%d (+%d)"
|
||||
% (th.user, th.descr, th.length, th.get_offset(), len(th))))
|
||||
print()
|
||||
th = th.prev_txn()
|
||||
i -= 1
|
||||
|
||||
def Main():
|
||||
ntxn = 10
|
||||
opts, args = getopt.getopt(sys.argv[1:], "n:")
|
||||
path, = args
|
||||
for k, v in opts:
|
||||
if k == '-n':
|
||||
ntxn = int(v)
|
||||
main(path, ntxn)
|
||||
|
||||
if __name__ == "__main__":
|
||||
Main()
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""Simple consistency checker for FileStorage.
|
||||
|
||||
usage: fstest.py [-v] data.fs
|
||||
|
||||
The fstest tool will scan all the data in a FileStorage and report an
|
||||
error if it finds any corrupt transaction data. The tool will print a
|
||||
message when the first error is detected, then exit.
|
||||
|
||||
The tool accepts one or more -v arguments. If a single -v is used, it
|
||||
will print a line of text for each transaction record it encounters.
|
||||
If two -v arguments are used, it will also print a line of text for
|
||||
each object. The objects for a transaction will be printed before the
|
||||
transaction itself.
|
||||
|
||||
Note: It does not check the consistency of the object pickles. It is
|
||||
possible for the damage to occur only in the part of the file that
|
||||
stores object pickles. Those errors will go undetected.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
# The implementation is based closely on the read_index() function in
|
||||
# ZODB.FileStorage. If anything about the FileStorage layout changes,
|
||||
# this file will need to be udpated.
|
||||
|
||||
import binascii
|
||||
import struct
|
||||
import sys
|
||||
from ZODB._compat import FILESTORAGE_MAGIC
|
||||
|
||||
class FormatError(ValueError):
|
||||
"""There is a problem with the format of the FileStorage."""
|
||||
|
||||
class Status(object):
|
||||
checkpoint = b'c'
|
||||
undone = b'u'
|
||||
|
||||
packed_version = FILESTORAGE_MAGIC
|
||||
|
||||
TREC_HDR_LEN = 23
|
||||
DREC_HDR_LEN = 42
|
||||
|
||||
VERBOSE = 0
|
||||
|
||||
def hexify(s):
|
||||
r"""Format an 8-bit string as hex
|
||||
|
||||
>>> hexify(b'\x00\xff\xaa\xcc')
|
||||
'0x00ffaacc'
|
||||
|
||||
"""
|
||||
return '0x' + binascii.hexlify(s).decode()
|
||||
|
||||
def chatter(msg, level=1):
|
||||
if VERBOSE >= level:
|
||||
sys.stdout.write(msg)
|
||||
|
||||
def U64(v):
|
||||
"""Unpack an 8-byte string as a 64-bit long"""
|
||||
h, l = struct.unpack(">II", v)
|
||||
if h:
|
||||
return (h << 32) + l
|
||||
else:
|
||||
return l
|
||||
|
||||
def check(path):
|
||||
with open(path, 'rb') as file:
|
||||
file.seek(0, 2)
|
||||
file_size = file.tell()
|
||||
if file_size == 0:
|
||||
raise FormatError("empty file")
|
||||
file.seek(0)
|
||||
if file.read(4) != packed_version:
|
||||
raise FormatError("invalid file header")
|
||||
|
||||
pos = 4
|
||||
tid = b'\000' * 8 # lowest possible tid to start
|
||||
i = 0
|
||||
while pos:
|
||||
_pos = pos
|
||||
pos, tid = check_trec(path, file, pos, tid, file_size)
|
||||
if tid is not None:
|
||||
chatter("%10d: transaction tid %s #%d \n" %
|
||||
(_pos, hexify(tid), i))
|
||||
i = i + 1
|
||||
|
||||
|
||||
def check_trec(path, file, pos, ltid, file_size):
|
||||
"""Read an individual transaction record from file.
|
||||
|
||||
Returns the pos of the next transaction and the transaction id.
|
||||
It also leaves the file pointer set to pos. The path argument is
|
||||
used for generating error messages.
|
||||
"""
|
||||
|
||||
h = file.read(TREC_HDR_LEN) #XXX must be bytes under Py3k
|
||||
if not h:
|
||||
return None, None
|
||||
if len(h) != TREC_HDR_LEN:
|
||||
raise FormatError("%s truncated at %s" % (path, pos))
|
||||
|
||||
tid, stl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
|
||||
tmeta_len = TREC_HDR_LEN + ul + dl + el
|
||||
|
||||
if tid <= ltid:
|
||||
raise FormatError("%s time-stamp reduction at %s: %s <= %s" %
|
||||
(path, pos, hexify(tid), hexify(ltid)))
|
||||
ltid = tid
|
||||
|
||||
tl = U64(stl) # transaction record length - 8
|
||||
if pos + tl + 8 > file_size:
|
||||
raise FormatError("%s truncated possibly because of"
|
||||
" damaged records at %s" % (path, pos))
|
||||
if status == Status.checkpoint:
|
||||
raise FormatError("%s checkpoint flag was not cleared at %s"
|
||||
% (path, pos))
|
||||
if status not in b' up':
|
||||
raise FormatError("%s has invalid status '%s' at %s" %
|
||||
(path, status, pos))
|
||||
|
||||
if tmeta_len > tl:
|
||||
raise FormatError("%s has an invalid transaction header"
|
||||
" at %s" % (path, pos))
|
||||
|
||||
tpos = pos
|
||||
tend = tpos + tl
|
||||
|
||||
if status != Status.undone:
|
||||
pos = tpos + tmeta_len
|
||||
file.read(ul + dl + el) # skip transaction metadata
|
||||
|
||||
i = 0
|
||||
while pos < tend:
|
||||
_pos = pos
|
||||
pos, oid = check_drec(path, file, pos, tpos, tid)
|
||||
if pos > tend:
|
||||
raise FormatError("%s has data records that extend beyond"
|
||||
" the transaction record; end at %s" %
|
||||
(path, pos))
|
||||
chatter("%10d: object oid %s #%d\n" % (_pos, hexify(oid), i),
|
||||
level=2)
|
||||
i = i + 1
|
||||
|
||||
file.seek(tend)
|
||||
rtl = file.read(8)
|
||||
if rtl != stl:
|
||||
raise FormatError("%s has inconsistent transaction length"
|
||||
" for undone transaction at %s" % (path, pos))
|
||||
pos = tend + 8
|
||||
return pos, tid
|
||||
|
||||
def check_drec(path, file, pos, tpos, tid):
|
||||
"""Check a data record for the current transaction record"""
|
||||
|
||||
h = file.read(DREC_HDR_LEN)
|
||||
if len(h) != DREC_HDR_LEN:
|
||||
raise FormatError("%s truncated at %s" % (path, pos))
|
||||
oid, serial, _prev, _tloc, vlen, _plen = (
|
||||
struct.unpack(">8s8s8s8sH8s", h))
|
||||
prev = U64(_prev)
|
||||
tloc = U64(_tloc)
|
||||
plen = U64(_plen)
|
||||
dlen = DREC_HDR_LEN + (plen or 8)
|
||||
|
||||
if vlen:
|
||||
dlen = dlen + 16 + vlen
|
||||
file.seek(8, 1)
|
||||
pv = U64(file.read(8))
|
||||
file.seek(vlen, 1) # skip the version data
|
||||
|
||||
if tloc != tpos:
|
||||
raise FormatError("%s data record exceeds transaction record "
|
||||
"at %s: tloc %d != tpos %d" %
|
||||
(path, pos, tloc, tpos))
|
||||
|
||||
pos = pos + dlen
|
||||
if plen:
|
||||
file.seek(plen, 1)
|
||||
else:
|
||||
file.seek(8, 1)
|
||||
# _loadBack() ?
|
||||
|
||||
return pos, oid
|
||||
|
||||
def usage():
|
||||
sys.exit(__doc__)
|
||||
|
||||
def main(args=None):
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
import getopt
|
||||
|
||||
global VERBOSE
|
||||
try:
|
||||
opts, args = getopt.getopt(args, 'v')
|
||||
if len(args) != 1:
|
||||
raise ValueError("expected one argument")
|
||||
for k, v in opts:
|
||||
if k == '-v':
|
||||
VERBOSE = VERBOSE + 1
|
||||
except (getopt.error, ValueError):
|
||||
usage()
|
||||
|
||||
try:
|
||||
check(args[0])
|
||||
except FormatError as msg:
|
||||
sys.exit(msg)
|
||||
|
||||
chatter("no errors detected")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
@@ -0,0 +1,177 @@
|
||||
"""Verify that fstest.py can find errors.
|
||||
|
||||
Note: To run this test script fstest.py must be on your PYTHONPATH.
|
||||
"""
|
||||
|
||||
from cStringIO import StringIO
|
||||
import re
|
||||
import struct
|
||||
import unittest
|
||||
import ZODB.tests.util
|
||||
|
||||
import fstest
|
||||
from fstest import FormatError, U64
|
||||
|
||||
class TestCorruptedFS(ZODB.tests.util.TestCase):
|
||||
|
||||
f = open('test-checker.fs', 'rb')
|
||||
datafs = f.read()
|
||||
f.close()
|
||||
del f
|
||||
|
||||
def setUp(self):
|
||||
ZODB.tests.util.TestCase.setUp(self)
|
||||
self._temp = 'Data.fs'
|
||||
self._file = open(self._temp, 'wb')
|
||||
|
||||
def tearDown(self):
|
||||
if not self._file.closed:
|
||||
self._file.close()
|
||||
ZODB.tests.util.TestCase.tearDown(self)
|
||||
|
||||
def noError(self):
|
||||
if not self._file.closed:
|
||||
self._file.close()
|
||||
fstest.check(self._temp)
|
||||
|
||||
def detectsError(self, rx):
|
||||
if not self._file.closed:
|
||||
self._file.close()
|
||||
try:
|
||||
fstest.check(self._temp)
|
||||
except FormatError as msg:
|
||||
mo = re.search(rx, str(msg))
|
||||
self.assertFalse(mo is None, "unexpected error: %s" % msg)
|
||||
else:
|
||||
self.fail("fstest did not detect corruption")
|
||||
|
||||
def getHeader(self):
|
||||
buf = self._datafs.read(16)
|
||||
if not buf:
|
||||
return 0, ''
|
||||
tl = U64(buf[8:])
|
||||
return tl, buf
|
||||
|
||||
def copyTransactions(self, n):
|
||||
"""Copy at most n transactions from the good data"""
|
||||
f = self._datafs = StringIO(self.datafs)
|
||||
self._file.write(f.read(4))
|
||||
for i in range(n):
|
||||
tl, data = self.getHeader()
|
||||
if not tl:
|
||||
return
|
||||
self._file.write(data)
|
||||
rec = f.read(tl - 8)
|
||||
self._file.write(rec)
|
||||
|
||||
def testGood(self):
|
||||
self._file.write(self.datafs)
|
||||
self.noError()
|
||||
|
||||
def testTwoTransactions(self):
|
||||
self.copyTransactions(2)
|
||||
self.noError()
|
||||
|
||||
def testEmptyFile(self):
|
||||
self.detectsError("empty file")
|
||||
|
||||
def testInvalidHeader(self):
|
||||
self._file.write('SF12')
|
||||
self.detectsError("invalid file header")
|
||||
|
||||
def testTruncatedTransaction(self):
|
||||
self._file.write(self.datafs[:4+22])
|
||||
self.detectsError("truncated")
|
||||
|
||||
def testCheckpointFlag(self):
|
||||
self.copyTransactions(2)
|
||||
tl, data = self.getHeader()
|
||||
assert tl > 0, "ran out of good transaction data"
|
||||
self._file.write(data)
|
||||
self._file.write('c')
|
||||
self._file.write(self._datafs.read(tl - 9))
|
||||
self.detectsError("checkpoint flag")
|
||||
|
||||
def testInvalidStatus(self):
|
||||
self.copyTransactions(2)
|
||||
tl, data = self.getHeader()
|
||||
assert tl > 0, "ran out of good transaction data"
|
||||
self._file.write(data)
|
||||
self._file.write('Z')
|
||||
self._file.write(self._datafs.read(tl - 9))
|
||||
self.detectsError("invalid status")
|
||||
|
||||
def testTruncatedRecord(self):
|
||||
self.copyTransactions(3)
|
||||
tl, data = self.getHeader()
|
||||
assert tl > 0, "ran out of good transaction data"
|
||||
self._file.write(data)
|
||||
buf = self._datafs.read(tl / 2)
|
||||
self._file.write(buf)
|
||||
self.detectsError("truncated possibly")
|
||||
|
||||
def testBadLength(self):
|
||||
self.copyTransactions(2)
|
||||
tl, data = self.getHeader()
|
||||
assert tl > 0, "ran out of good transaction data"
|
||||
self._file.write(data)
|
||||
buf = self._datafs.read(tl - 8)
|
||||
self._file.write(buf[0])
|
||||
assert tl <= 1<<16, "can't use this transaction for this test"
|
||||
self._file.write("\777\777")
|
||||
self._file.write(buf[3:])
|
||||
self.detectsError("invalid transaction header")
|
||||
|
||||
def testDecreasingTimestamps(self):
|
||||
self.copyTransactions(0)
|
||||
tl, data = self.getHeader()
|
||||
buf = self._datafs.read(tl - 8)
|
||||
t1 = data + buf
|
||||
|
||||
tl, data = self.getHeader()
|
||||
buf = self._datafs.read(tl - 8)
|
||||
t2 = data + buf
|
||||
|
||||
self._file.write(t2[:8] + t1[8:])
|
||||
self._file.write(t1[:8] + t2[8:])
|
||||
self.detectsError("time-stamp")
|
||||
|
||||
def testTruncatedData(self):
|
||||
# This test must re-write the transaction header length in
|
||||
# order to trigger the error in check_drec(). If it doesn't,
|
||||
# the truncated data record would also caught a truncated
|
||||
# transaction record.
|
||||
self.copyTransactions(1)
|
||||
tl, data = self.getHeader()
|
||||
pos = self._file.tell()
|
||||
self._file.write(data)
|
||||
buf = self._datafs.read(tl - 8)
|
||||
hdr = buf[:15]
|
||||
ul, dl, el = struct.unpack(">HHH", hdr[-6:])
|
||||
self._file.write(buf[:15 + ul + dl + el])
|
||||
data = buf[15 + ul + dl + el:]
|
||||
self._file.write(data[:24])
|
||||
self._file.seek(pos + 8, 0)
|
||||
newlen = struct.pack(">II", 0, tl - (len(data) - 24))
|
||||
self._file.write(newlen)
|
||||
self.detectsError("truncated at")
|
||||
|
||||
def testBadDataLength(self):
|
||||
self.copyTransactions(1)
|
||||
tl, data = self.getHeader()
|
||||
self._file.write(data)
|
||||
buf = self._datafs.read(tl - 8)
|
||||
hdr = buf[:7]
|
||||
# write the transaction meta data
|
||||
ul, dl, el = struct.unpack(">HHH", hdr[-6:])
|
||||
self._file.write(buf[:7 + ul + dl + el])
|
||||
|
||||
# write the first part of the data header
|
||||
data = buf[7 + ul + dl + el:]
|
||||
self._file.write(data[:24])
|
||||
self._file.write("\000" * 4 + "\077" + "\000" * 3)
|
||||
self._file.write(data[32:])
|
||||
self.detectsError("record exceeds transaction")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,358 @@
|
||||
#!/usr/bin/env python
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002, 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""A script to gather statistics while doing a storage migration.
|
||||
|
||||
This is very similar to a standard storage's copyTransactionsFrom() method,
|
||||
except that it's geared to run as a script, and it collects useful pieces of
|
||||
information as it's working. This script can be used to stress test a storage
|
||||
since it blasts transactions at it as fast as possible. You can get a good
|
||||
sense of the performance of a storage by running this script.
|
||||
|
||||
Actually it just counts the size of pickles in the transaction via the
|
||||
iterator protocol, so storage overheads aren't counted.
|
||||
|
||||
Usage: %(PROGRAM)s [options] [source-storage-args] [destination-storage-args]
|
||||
Options:
|
||||
-S sourcetype
|
||||
--stype=sourcetype
|
||||
This is the name of a recognized type for the source database. Use -T
|
||||
to print out the known types. Defaults to "file".
|
||||
|
||||
-D desttype
|
||||
--dtype=desttype
|
||||
This is the name of the recognized type for the destination database.
|
||||
Use -T to print out the known types. Defaults to "file".
|
||||
|
||||
-o filename
|
||||
--output=filename
|
||||
Print results in filename, otherwise stdout.
|
||||
|
||||
-m txncount
|
||||
--max=txncount
|
||||
Stop after committing txncount transactions.
|
||||
|
||||
-k txncount
|
||||
--skip=txncount
|
||||
Skip the first txncount transactions.
|
||||
|
||||
-p/--profile
|
||||
Turn on specialized profiling.
|
||||
|
||||
-t/--timestamps
|
||||
Print tids as timestamps.
|
||||
|
||||
-T/--storage_types
|
||||
Print all the recognized storage types and exit.
|
||||
|
||||
-v/--verbose
|
||||
Turns on verbose output. Multiple -v options increase the verbosity.
|
||||
|
||||
-h/--help
|
||||
Print this message and exit.
|
||||
|
||||
Positional arguments:
|
||||
|
||||
source-storage-args:
|
||||
Semicolon separated list of arguments for the source storage, as
|
||||
key=val pairs. E.g. "file_name=Data.fs;read_only=1"
|
||||
|
||||
destination-storage-args:
|
||||
Comma separated list of arguments for the source storage, as key=val
|
||||
pairs. E.g. "name=full;frequency=3600"
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import getopt
|
||||
import marshal
|
||||
import profile
|
||||
|
||||
from ZODB import utils
|
||||
from ZODB import StorageTypes
|
||||
from ZODB.TimeStamp import TimeStamp
|
||||
|
||||
PROGRAM = sys.argv[0]
|
||||
ZERO = '\0'*8
|
||||
|
||||
|
||||
def usage(code, msg=''):
|
||||
print(__doc__ % globals(), file=sys.stderr)
|
||||
if msg:
|
||||
print(msg, file=sys.stderr)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def error(code, msg):
|
||||
print(msg, file=sys.stderr)
|
||||
print("use --help for usage message")
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
opts, args = getopt.getopt(
|
||||
sys.argv[1:],
|
||||
'hvo:pm:k:D:S:Tt',
|
||||
['help', 'verbose',
|
||||
'output=', 'profile', 'storage_types',
|
||||
'max=', 'skip=', 'dtype=', 'stype=', 'timestamps'])
|
||||
except getopt.error as msg:
|
||||
error(2, msg)
|
||||
|
||||
class Options(object):
|
||||
stype = 'FileStorage'
|
||||
dtype = 'FileStorage'
|
||||
verbose = 0
|
||||
outfile = None
|
||||
profilep = False
|
||||
maxtxn = -1
|
||||
skiptxn = -1
|
||||
timestamps = False
|
||||
|
||||
options = Options()
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt in ('-h', '--help'):
|
||||
usage(0)
|
||||
elif opt in ('-v', '--verbose'):
|
||||
options.verbose += 1
|
||||
elif opt in ('-T', '--storage_types'):
|
||||
print_types()
|
||||
sys.exit(0)
|
||||
elif opt in ('-S', '--stype'):
|
||||
options.stype = arg
|
||||
elif opt in ('-D', '--dtype'):
|
||||
options.dtype = arg
|
||||
elif opt in ('-o', '--output'):
|
||||
options.outfile = arg
|
||||
elif opt in ('-p', '--profile'):
|
||||
options.profilep = True
|
||||
elif opt in ('-m', '--max'):
|
||||
options.maxtxn = int(arg)
|
||||
elif opt in ('-k', '--skip'):
|
||||
options.skiptxn = int(arg)
|
||||
elif opt in ('-t', '--timestamps'):
|
||||
options.timestamps = True
|
||||
|
||||
if len(args) > 2:
|
||||
error(2, "too many arguments")
|
||||
|
||||
srckws = {}
|
||||
if len(args) > 0:
|
||||
srcargs = args[0]
|
||||
for kv in re.split(r';\s*', srcargs):
|
||||
key, val = kv.split('=')
|
||||
srckws[key] = val
|
||||
|
||||
destkws = {}
|
||||
if len(args) > 1:
|
||||
destargs = args[1]
|
||||
for kv in re.split(r';\s*', destargs):
|
||||
key, val = kv.split('=')
|
||||
destkws[key] = val
|
||||
|
||||
if options.stype not in StorageTypes.storage_types.keys():
|
||||
usage(2, 'Source database type must be provided')
|
||||
if options.dtype not in StorageTypes.storage_types.keys():
|
||||
usage(2, 'Destination database type must be provided')
|
||||
|
||||
# Open the output file
|
||||
if options.outfile is None:
|
||||
options.outfp = sys.stdout
|
||||
options.outclosep = False
|
||||
else:
|
||||
options.outfp = open(options.outfile, 'w')
|
||||
options.outclosep = True
|
||||
|
||||
if options.verbose > 0:
|
||||
print('Opening source database...')
|
||||
modname, sconv = StorageTypes.storage_types[options.stype]
|
||||
kw = sconv(**srckws)
|
||||
__import__(modname)
|
||||
sclass = getattr(sys.modules[modname], options.stype)
|
||||
srcdb = sclass(**kw)
|
||||
|
||||
if options.verbose > 0:
|
||||
print('Opening destination database...')
|
||||
modname, dconv = StorageTypes.storage_types[options.dtype]
|
||||
kw = dconv(**destkws)
|
||||
__import__(modname)
|
||||
dclass = getattr(sys.modules[modname], options.dtype)
|
||||
dstdb = dclass(**kw)
|
||||
|
||||
try:
|
||||
t0 = time.time()
|
||||
doit(srcdb, dstdb, options)
|
||||
t1 = time.time()
|
||||
if options.verbose > 0:
|
||||
print('Migration time: %8.3f' % (t1-t0))
|
||||
finally:
|
||||
# Done
|
||||
srcdb.close()
|
||||
dstdb.close()
|
||||
if options.outclosep:
|
||||
options.outfp.close()
|
||||
|
||||
|
||||
def doit(srcdb, dstdb, options):
|
||||
outfp = options.outfp
|
||||
profilep = options.profilep
|
||||
verbose = options.verbose
|
||||
# some global information
|
||||
largest_pickle = 0
|
||||
largest_txn_in_size = 0
|
||||
largest_txn_in_objects = 0
|
||||
total_pickle_size = 0
|
||||
total_object_count = 0
|
||||
# Ripped from BaseStorage.copyTransactionsFrom()
|
||||
ts = None
|
||||
ok = True
|
||||
prevrevids = {}
|
||||
counter = 0
|
||||
skipper = 0
|
||||
if options.timestamps:
|
||||
print("%4s. %26s %6s %8s %5s %5s %5s %5s %5s" % (
|
||||
"NUM", "TID AS TIMESTAMP", "OBJS", "BYTES",
|
||||
# Does anybody know what these times mean?
|
||||
"t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3"))
|
||||
else:
|
||||
print("%4s. %20s %6s %8s %6s %6s %6s %6s %6s" % (
|
||||
"NUM", "TRANSACTION ID", "OBJS", "BYTES",
|
||||
# Does anybody know what these times mean?
|
||||
"t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3"))
|
||||
for txn in srcdb.iterator():
|
||||
skipper += 1
|
||||
if skipper <= options.skiptxn:
|
||||
continue
|
||||
counter += 1
|
||||
if counter > options.maxtxn >= 0:
|
||||
break
|
||||
tid = txn.tid
|
||||
if ts is None:
|
||||
ts = TimeStamp(tid)
|
||||
else:
|
||||
t = TimeStamp(tid)
|
||||
if t <= ts:
|
||||
if ok:
|
||||
print((
|
||||
'Time stamps are out of order %s, %s' % (ts, t)), file=sys.stderr)
|
||||
ok = False
|
||||
ts = t.laterThan(ts)
|
||||
tid = ts.raw()
|
||||
else:
|
||||
ts = t
|
||||
if not ok:
|
||||
print((
|
||||
'Time stamps are back in order %s' % t), file=sys.stderr)
|
||||
ok = True
|
||||
if verbose > 1:
|
||||
print(ts)
|
||||
|
||||
prof = None
|
||||
if profilep and (counter % 100) == 0:
|
||||
prof = profile.Profile()
|
||||
objects = 0
|
||||
size = 0
|
||||
newrevids = RevidAccumulator()
|
||||
t0 = time.time()
|
||||
dstdb.tpc_begin(txn, tid, txn.status)
|
||||
t1 = time.time()
|
||||
for r in txn:
|
||||
oid = r.oid
|
||||
objects += 1
|
||||
thissize = len(r.data)
|
||||
size += thissize
|
||||
if thissize > largest_pickle:
|
||||
largest_pickle = thissize
|
||||
if verbose > 1:
|
||||
if not r.version:
|
||||
vstr = 'norev'
|
||||
else:
|
||||
vstr = r.version
|
||||
print(utils.U64(oid), vstr, len(r.data))
|
||||
oldrevid = prevrevids.get(oid, ZERO)
|
||||
result = dstdb.store(oid, oldrevid, r.data, r.version, txn)
|
||||
newrevids.store(oid, result)
|
||||
t2 = time.time()
|
||||
result = dstdb.tpc_vote(txn)
|
||||
t3 = time.time()
|
||||
newrevids.tpc_vote(result)
|
||||
prevrevids.update(newrevids.get_dict())
|
||||
# Profile every 100 transactions
|
||||
if prof:
|
||||
prof.runcall(dstdb.tpc_finish, txn)
|
||||
else:
|
||||
dstdb.tpc_finish(txn)
|
||||
t4 = time.time()
|
||||
|
||||
# record the results
|
||||
if objects > largest_txn_in_objects:
|
||||
largest_txn_in_objects = objects
|
||||
if size > largest_txn_in_size:
|
||||
largest_txn_in_size = size
|
||||
if options.timestamps:
|
||||
tidstr = str(TimeStamp(tid))
|
||||
format = "%4d. %26s %6d %8d %5.3f %5.3f %5.3f %5.3f %5.3f"
|
||||
else:
|
||||
tidstr = utils.U64(tid)
|
||||
format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f"
|
||||
print(format % (skipper, tidstr, objects, size,
|
||||
t4-t0, t1-t0, t2-t1, t3-t2, t4-t3), file=outfp)
|
||||
total_pickle_size += size
|
||||
total_object_count += objects
|
||||
|
||||
if prof:
|
||||
prof.create_stats()
|
||||
fp = open('profile-%02d.txt' % (counter / 100), 'wb')
|
||||
marshal.dump(prof.stats, fp)
|
||||
fp.close()
|
||||
print("Largest pickle: %8d" % largest_pickle, file=outfp)
|
||||
print("Largest transaction: %8d" % largest_txn_in_size, file=outfp)
|
||||
print("Largest object count: %8d" % largest_txn_in_objects, file=outfp)
|
||||
print("Total pickle size: %14d" % total_pickle_size, file=outfp)
|
||||
print("Total object count: %8d" % total_object_count, file=outfp)
|
||||
|
||||
|
||||
# helper to deal with differences between old-style store() return and
|
||||
# new-style store() return that supports ZEO
|
||||
|
||||
class RevidAccumulator(object):
|
||||
|
||||
def __init__(self):
|
||||
self.data = {}
|
||||
|
||||
def _update_from_list(self, list):
|
||||
for oid, serial in list:
|
||||
if not isinstance(serial, str):
|
||||
raise serial
|
||||
self.data[oid] = serial
|
||||
|
||||
def store(self, oid, result):
|
||||
if isinstance(result, str):
|
||||
self.data[oid] = result
|
||||
elif result is not None:
|
||||
self._update_from_list(result)
|
||||
|
||||
def tpc_vote(self, result):
|
||||
if result is not None:
|
||||
self._update_from_list(result)
|
||||
|
||||
def get_dict(self):
|
||||
return self.data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2008 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""A script to migrate a blob directory into a different layout.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from ZODB.blob import FilesystemHelper
|
||||
from ZODB.utils import oid_repr
|
||||
|
||||
|
||||
def link_or_copy(f1, f2):
|
||||
try:
|
||||
os.link(f1, f2)
|
||||
except OSError:
|
||||
shutil.copy(f1, f2)
|
||||
|
||||
# Check if we actually have link
|
||||
try:
|
||||
os.link
|
||||
except AttributeError:
|
||||
link_or_copy = shutil.copy
|
||||
|
||||
|
||||
def migrate(source, dest, layout):
|
||||
source_fsh = FilesystemHelper(source)
|
||||
source_fsh.create()
|
||||
dest_fsh = FilesystemHelper(dest, layout)
|
||||
dest_fsh.create()
|
||||
print("Migrating blob data from `%s` (%s) to `%s` (%s)" % (
|
||||
source, source_fsh.layout_name, dest, dest_fsh.layout_name))
|
||||
for oid, path in source_fsh.listOIDs():
|
||||
dest_path = dest_fsh.getPathForOID(oid, create=True)
|
||||
files = os.listdir(path)
|
||||
for file in files:
|
||||
source_file = os.path.join(path, file)
|
||||
dest_file = os.path.join(dest_path, file)
|
||||
link_or_copy(source_file, dest_file)
|
||||
print("\tOID: %s - %s files " % (oid_repr(oid), len(files)))
|
||||
|
||||
|
||||
def main(source=None, dest=None, layout="bushy"):
|
||||
usage = "usage: %prog [options] <source> <dest> <layout>"
|
||||
description = ("Create the new directory <dest> and migrate all blob "
|
||||
"data <source> to <dest> while using the new <layout> for "
|
||||
"<dest>")
|
||||
|
||||
parser = optparse.OptionParser(usage=usage, description=description)
|
||||
parser.add_option("-l", "--layout",
|
||||
default=layout, type='choice',
|
||||
choices=['bushy', 'lawn'],
|
||||
help="Define the layout to use for the new directory "
|
||||
"(bushy or lawn). Default: %default")
|
||||
options, args = parser.parse_args()
|
||||
|
||||
if not len(args) == 2:
|
||||
parser.error("source and destination must be given")
|
||||
|
||||
logging.getLogger().addHandler(logging.StreamHandler())
|
||||
logging.getLogger().setLevel(0)
|
||||
|
||||
source, dest = args
|
||||
migrate(source, dest, options.layout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python
|
||||
"""Report on the net size of objects counting subobjects.
|
||||
|
||||
usage: netspace.py [-P | -v] data.fs
|
||||
|
||||
-P: do a pack first
|
||||
-v: print info for all objects, even if a traversal path isn't found
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import ZODB
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from ZODB.utils import U64, get_pickle_metadata, load_current
|
||||
from ZODB.serialize import referencesf
|
||||
from six.moves import filter
|
||||
|
||||
def find_paths(root, maxdist):
|
||||
"""Find Python attribute traversal paths for objects to maxdist distance.
|
||||
|
||||
Starting at a root object, traverse attributes up to distance levels
|
||||
from the root, looking for persistent objects. Return a dict
|
||||
mapping oids to traversal paths.
|
||||
|
||||
TODO: Assumes that the keys of the root are not themselves
|
||||
persistent objects.
|
||||
|
||||
TODO: Doesn't traverse containers.
|
||||
"""
|
||||
paths = {}
|
||||
|
||||
# Handle the root as a special case because it's a dict
|
||||
objs = []
|
||||
for k, v in root.items():
|
||||
oid = getattr(v, '_p_oid', None)
|
||||
objs.append((k, v, oid, 0))
|
||||
|
||||
for path, obj, oid, dist in objs:
|
||||
if oid is not None:
|
||||
paths[oid] = path
|
||||
if dist < maxdist:
|
||||
getattr(obj, 'foo', None) # unghostify
|
||||
try:
|
||||
items = obj.__dict__.items()
|
||||
except AttributeError:
|
||||
continue
|
||||
for k, v in items:
|
||||
oid = getattr(v, '_p_oid', None)
|
||||
objs.append(("%s.%s" % (path, k), v, oid, dist + 1))
|
||||
|
||||
return paths
|
||||
|
||||
def main(path):
|
||||
fs = FileStorage(path, read_only=1)
|
||||
if PACK:
|
||||
fs.pack()
|
||||
|
||||
db = ZODB.DB(fs)
|
||||
rt = db.open().root()
|
||||
paths = find_paths(rt, 3)
|
||||
|
||||
def total_size(oid):
|
||||
cache = {}
|
||||
cache_size = 1000
|
||||
def _total_size(oid, seen):
|
||||
v = cache.get(oid)
|
||||
if v is not None:
|
||||
return v
|
||||
data, serialno = load_current(fs, oid)
|
||||
size = len(data)
|
||||
for suboid in referencesf(data):
|
||||
if suboid in seen:
|
||||
continue
|
||||
seen[suboid] = 1
|
||||
size += _total_size(suboid, seen)
|
||||
cache[oid] = size
|
||||
if len(cache) == cache_size:
|
||||
cache.popitem()
|
||||
return size
|
||||
return _total_size(oid, {})
|
||||
|
||||
keys = fs._index.keys()
|
||||
keys.sort()
|
||||
keys.reverse()
|
||||
|
||||
if not VERBOSE:
|
||||
# If not running verbosely, don't print an entry for an object
|
||||
# unless it has an entry in paths.
|
||||
keys = filter(paths.has_key, keys)
|
||||
|
||||
fmt = "%8s %5d %8d %s %s.%s"
|
||||
|
||||
for oid in keys:
|
||||
data, serialno = load_current(fs, oid)
|
||||
mod, klass = get_pickle_metadata(data)
|
||||
refs = referencesf(data)
|
||||
path = paths.get(oid, '-')
|
||||
print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass))
|
||||
|
||||
def Main():
|
||||
import sys
|
||||
import getopt
|
||||
|
||||
global PACK
|
||||
global VERBOSE
|
||||
|
||||
PACK = 0
|
||||
VERBOSE = 0
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'Pv')
|
||||
path, = args
|
||||
except getopt.error as err:
|
||||
print(err)
|
||||
print(__doc__)
|
||||
sys.exit(2)
|
||||
except ValueError:
|
||||
print("expected one argument, got", len(args))
|
||||
print(__doc__)
|
||||
sys.exit(2)
|
||||
for o, v in opts:
|
||||
if o == '-P':
|
||||
PACK = 1
|
||||
if o == '-v':
|
||||
VERBOSE += 1
|
||||
main(path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
Main()
|
||||
@@ -0,0 +1,27 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2005 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Compute a table of object id referrers
|
||||
|
||||
$Id$
|
||||
"""
|
||||
|
||||
from ZODB.serialize import referencesf
|
||||
|
||||
def referrers(storage):
|
||||
result = {}
|
||||
for transaction in storage.iterator():
|
||||
for record in transaction:
|
||||
for oid in referencesf(record.data):
|
||||
result.setdefault(oid, []).append((record.oid, record.tid))
|
||||
return result
|
||||
@@ -0,0 +1,747 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# repozo.py -- incremental and full backups of a Data.fs file.
|
||||
#
|
||||
# Originally written by Anthony Baxter
|
||||
# Significantly modified by Barry Warsaw
|
||||
|
||||
"""repozo.py -- incremental and full backups of a Data.fs file and index.
|
||||
|
||||
Usage: %(program)s [options]
|
||||
Where:
|
||||
|
||||
Exactly one of -B, -R, or -V must be specified:
|
||||
|
||||
-B / --backup
|
||||
Backup current ZODB file.
|
||||
|
||||
-R / --recover
|
||||
Restore a ZODB file from a backup.
|
||||
|
||||
-V / --verify
|
||||
Verify backup integrity.
|
||||
|
||||
-v / --verbose
|
||||
Verbose mode.
|
||||
|
||||
-h / --help
|
||||
Print this text and exit.
|
||||
|
||||
-r dir
|
||||
--repository=dir
|
||||
Repository directory containing the backup files. This argument
|
||||
is required. The directory must already exist. You should not
|
||||
edit the files in this directory, or add your own files to it.
|
||||
|
||||
Options for -B/--backup:
|
||||
-f file
|
||||
--file=file
|
||||
Source Data.fs file. This argument is required.
|
||||
|
||||
-F / --full
|
||||
Force a full backup. By default, an incremental backup is made
|
||||
if possible (e.g., if a pack has occurred since the last
|
||||
incremental backup, a full backup is necessary).
|
||||
|
||||
-Q / --quick
|
||||
Verify via md5 checksum only the last incremental written. This
|
||||
significantly reduces the disk i/o at the (theoretical) cost of
|
||||
inconsistency. This is a probabilistic way of determining whether
|
||||
a full backup is necessary.
|
||||
|
||||
-z / --gzip
|
||||
Compress with gzip the backup files. Uses the default zlib
|
||||
compression level. By default, gzip compression is not used.
|
||||
|
||||
-k / --kill-old-on-full
|
||||
If a full backup is created, remove any prior full or incremental
|
||||
backup files (and associated metadata files) from the repository
|
||||
directory.
|
||||
|
||||
Options for -R/--recover:
|
||||
-D str
|
||||
--date=str
|
||||
Recover state as of this date. Specify UTC (not local) time.
|
||||
yyyy-mm-dd[-hh[-mm[-ss]]]
|
||||
By default, current time is used.
|
||||
|
||||
-o filename
|
||||
--output=filename
|
||||
Write recovered ZODB to given file. By default, the file is
|
||||
written to stdout.
|
||||
|
||||
Note: for the stdout case, the index file will **not** be restored
|
||||
automatically.
|
||||
|
||||
Options for -V/--verify:
|
||||
-Q / --quick
|
||||
Verify file sizes only (skip md5 checksums).
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from six.moves import filter
|
||||
from hashlib import md5
|
||||
import gzip
|
||||
import time
|
||||
import errno
|
||||
import getopt
|
||||
|
||||
from ZODB.FileStorage import FileStorage
|
||||
|
||||
program = sys.argv[0]
|
||||
|
||||
BACKUP = 1
|
||||
RECOVER = 2
|
||||
VERIFY = 3
|
||||
|
||||
COMMASPACE = ', '
|
||||
READCHUNK = 16 * 1024
|
||||
VERBOSE = False
|
||||
|
||||
|
||||
class WouldOverwriteFiles(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoFiles(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _GzipCloser(object):
|
||||
|
||||
def __init__(self, fqn, mode):
|
||||
self._opened = gzip.open(fqn, mode)
|
||||
|
||||
def __enter__(self):
|
||||
return self._opened
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self._opened.close()
|
||||
|
||||
|
||||
def usage(code, msg=''):
|
||||
outfp = sys.stderr
|
||||
if code == 0:
|
||||
outfp = sys.stdout
|
||||
|
||||
print(__doc__ % globals(), file=outfp)
|
||||
if msg:
|
||||
print(msg, file=outfp)
|
||||
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def log(msg, *args):
|
||||
if VERBOSE:
|
||||
# Use stderr here so that -v flag works with -R and no -o
|
||||
print(msg % args, file=sys.stderr)
|
||||
|
||||
|
||||
def error(msg, *args):
|
||||
print(msg % args, file=sys.stderr)
|
||||
|
||||
|
||||
def parseargs(argv):
|
||||
global VERBOSE
|
||||
try:
|
||||
opts, args = getopt.getopt(argv, 'BRVvhr:f:FQzkD:o:',
|
||||
['backup',
|
||||
'recover',
|
||||
'verify',
|
||||
'verbose',
|
||||
'help',
|
||||
'repository=',
|
||||
'file=',
|
||||
'full',
|
||||
'quick',
|
||||
'gzip',
|
||||
'kill-old-on-full',
|
||||
'date=',
|
||||
'output=',
|
||||
])
|
||||
except getopt.error as msg:
|
||||
usage(1, msg)
|
||||
|
||||
class Options(object):
|
||||
mode = None # BACKUP, RECOVER or VERIFY
|
||||
file = None # name of input Data.fs file
|
||||
repository = None # name of directory holding backups
|
||||
full = False # True forces full backup
|
||||
date = None # -D argument, if any
|
||||
output = None # where to write recovered data; None = stdout
|
||||
quick = False # -Q flag state
|
||||
gzip = False # -z flag state
|
||||
killold = False # -k flag state
|
||||
|
||||
options = Options()
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt in ('-h', '--help'):
|
||||
usage(0)
|
||||
elif opt in ('-v', '--verbose'):
|
||||
VERBOSE = True
|
||||
elif opt in ('-R', '--recover'):
|
||||
if options.mode is not None:
|
||||
usage(1, '-B, -R, and -V are mutually exclusive')
|
||||
options.mode = RECOVER
|
||||
elif opt in ('-B', '--backup'):
|
||||
if options.mode is not None:
|
||||
usage(1, '-B, -R, and -V are mutually exclusive')
|
||||
options.mode = BACKUP
|
||||
elif opt in ('-V', '--verify'):
|
||||
if options.mode is not None:
|
||||
usage(1, '-B, -R, and -V are mutually exclusive')
|
||||
options.mode = VERIFY
|
||||
elif opt in ('-Q', '--quick'):
|
||||
options.quick = True
|
||||
elif opt in ('-f', '--file'):
|
||||
options.file = arg
|
||||
elif opt in ('-r', '--repository'):
|
||||
options.repository = arg
|
||||
elif opt in ('-F', '--full'):
|
||||
options.full = True
|
||||
elif opt in ('-D', '--date'):
|
||||
options.date = arg
|
||||
elif opt in ('-o', '--output'):
|
||||
options.output = arg
|
||||
elif opt in ('-z', '--gzip'):
|
||||
options.gzip = True
|
||||
elif opt in ('-k', '--kill-old-on-full'):
|
||||
options.killold = True
|
||||
else:
|
||||
assert False, (opt, arg)
|
||||
|
||||
# Any other arguments are invalid
|
||||
if args:
|
||||
usage(1, 'Invalid arguments: ' + COMMASPACE.join(args))
|
||||
|
||||
# Sanity checks
|
||||
if options.mode is None:
|
||||
usage(1, 'Either --backup, --recover or --verify is required')
|
||||
if options.repository is None:
|
||||
usage(1, '--repository is required')
|
||||
if options.mode == BACKUP:
|
||||
if options.date is not None:
|
||||
log('--date option is ignored in backup mode')
|
||||
options.date = None
|
||||
if options.output is not None:
|
||||
log('--output option is ignored in backup mode')
|
||||
options.output = None
|
||||
elif options.mode == RECOVER:
|
||||
if options.file is not None:
|
||||
log('--file option is ignored in recover mode')
|
||||
options.file = None
|
||||
if options.killold:
|
||||
log('--kill-old-on-full option is ignored in recover mode')
|
||||
options.killold = False
|
||||
else:
|
||||
assert options.mode == VERIFY
|
||||
if options.date is not None:
|
||||
log("--date option is ignored in verify mode")
|
||||
options.date = None
|
||||
if options.output is not None:
|
||||
log('--output option is ignored in verify mode')
|
||||
options.output = None
|
||||
if options.full:
|
||||
log('--full option is ignored in verify mode')
|
||||
options.full = False
|
||||
if options.gzip:
|
||||
log('--gzip option is ignored in verify mode')
|
||||
options.gzip = False
|
||||
if options.file is not None:
|
||||
log('--file option is ignored in verify mode')
|
||||
options.file = None
|
||||
if options.killold:
|
||||
log('--kill-old-on-full option is ignored in verify mode')
|
||||
options.killold = False
|
||||
return options
|
||||
|
||||
|
||||
# afile is a Python file object, or created by gzip.open(). The latter
|
||||
# doesn't have a fileno() method, so to fsync it we need to reach into
|
||||
# its underlying file object.
|
||||
def fsync(afile):
|
||||
afile.flush()
|
||||
fileobject = getattr(afile, 'fileobj', afile)
|
||||
os.fsync(fileobject.fileno())
|
||||
|
||||
# Read bytes (no more than n, or to EOF if n is None) in chunks from the
|
||||
# current position in file fp. Pass each chunk as an argument to func().
|
||||
# Return the total number of bytes read == the total number of bytes
|
||||
# passed in all to func(). Leaves the file position just after the
|
||||
# last byte read.
|
||||
def dofile(func, fp, n=None):
|
||||
bytesread = 0
|
||||
while n is None or n > 0:
|
||||
if n is None:
|
||||
todo = READCHUNK
|
||||
else:
|
||||
todo = min(READCHUNK, n)
|
||||
data = fp.read(todo)
|
||||
if not data:
|
||||
break
|
||||
func(data)
|
||||
nread = len(data)
|
||||
bytesread += nread
|
||||
if n is not None:
|
||||
n -= nread
|
||||
return bytesread
|
||||
|
||||
|
||||
def checksum(fp, n):
|
||||
# Checksum the first n bytes of the specified file
|
||||
sum = md5()
|
||||
def func(data):
|
||||
sum.update(data)
|
||||
dofile(func, fp, n)
|
||||
return sum.hexdigest()
|
||||
|
||||
|
||||
def file_size(fp):
|
||||
# Compute number of bytes that can be read from fp
|
||||
def func(data):
|
||||
pass
|
||||
return dofile(func, fp, None)
|
||||
|
||||
|
||||
def checksum_and_size(fp):
|
||||
# Checksum and return it with the size of the file
|
||||
sum = md5()
|
||||
def func(data):
|
||||
sum.update(data)
|
||||
size = dofile(func, fp, None)
|
||||
return sum.hexdigest(), size
|
||||
|
||||
|
||||
def copyfile(options, dst, start, n):
|
||||
# Copy bytes from file src, to file dst, starting at offset start, for n
|
||||
# length of bytes. For robustness, we first write, flush and fsync
|
||||
# to a temp file, then rename the temp file at the end.
|
||||
sum = md5()
|
||||
ifp = open(options.file, 'rb')
|
||||
ifp.seek(start)
|
||||
tempname = os.path.join(os.path.dirname(dst), 'tmp.tmp')
|
||||
if options.gzip:
|
||||
ofp = gzip.open(tempname, 'wb')
|
||||
else:
|
||||
ofp = open(tempname, 'wb')
|
||||
|
||||
def func(data):
|
||||
sum.update(data)
|
||||
ofp.write(data)
|
||||
|
||||
ndone = dofile(func, ifp, n)
|
||||
assert ndone == n
|
||||
|
||||
ifp.close()
|
||||
fsync(ofp)
|
||||
ofp.close()
|
||||
os.rename(tempname, dst)
|
||||
return sum.hexdigest()
|
||||
|
||||
|
||||
def concat(files, ofp=None):
|
||||
# Concatenate a bunch of files from the repository, output to 'ofp' if
|
||||
# given. Return the number of bytes written and the md5 checksum of the
|
||||
# bytes.
|
||||
sum = md5()
|
||||
def func(data):
|
||||
sum.update(data)
|
||||
if ofp:
|
||||
ofp.write(data)
|
||||
bytesread = 0
|
||||
for f in files:
|
||||
# Auto uncompress
|
||||
if f.endswith('fsz'):
|
||||
ifp = gzip.open(f, 'rb')
|
||||
else:
|
||||
ifp = open(f, 'rb')
|
||||
bytesread += dofile(func, ifp)
|
||||
ifp.close()
|
||||
if ofp:
|
||||
ofp.close()
|
||||
return bytesread, sum.hexdigest()
|
||||
|
||||
|
||||
def gen_filedate(options):
|
||||
return getattr(options, 'test_now', time.gmtime()[:6])
|
||||
|
||||
def gen_filename(options, ext=None, now=None):
|
||||
if ext is None:
|
||||
if options.full:
|
||||
ext = '.fs'
|
||||
else:
|
||||
ext = '.deltafs'
|
||||
if options.gzip:
|
||||
ext += 'z'
|
||||
# Hook for testing
|
||||
if now is None:
|
||||
now = gen_filedate(options)
|
||||
t = now + (ext,)
|
||||
return '%04d-%02d-%02d-%02d-%02d-%02d%s' % t
|
||||
|
||||
# Return a list of files needed to reproduce state at time options.date.
|
||||
# This is a list, in chronological order, of the .fs[z] and .deltafs[z]
|
||||
# files, from the time of the most recent full backup preceding
|
||||
# options.date, up to options.date.
|
||||
|
||||
import re
|
||||
is_data_file = re.compile(r'\d{4}(?:-\d\d){5}\.(?:delta)?fsz?$').match
|
||||
del re
|
||||
|
||||
def find_files(options):
|
||||
when = options.date
|
||||
if not when:
|
||||
when = gen_filename(options, ext='')
|
||||
log('looking for files between last full backup and %s...', when)
|
||||
# newest file first
|
||||
all = sorted(
|
||||
filter(is_data_file, os.listdir(options.repository)), reverse=True)
|
||||
# Find the last full backup before date, then include all the
|
||||
# incrementals between that full backup and "when".
|
||||
needed = []
|
||||
for fname in all:
|
||||
root, ext = os.path.splitext(fname)
|
||||
if root <= when:
|
||||
needed.append(fname)
|
||||
if ext in ('.fs', '.fsz'):
|
||||
break
|
||||
# Make the file names relative to the repository directory
|
||||
needed = [os.path.join(options.repository, f) for f in needed]
|
||||
# Restore back to chronological order
|
||||
needed.reverse()
|
||||
if needed:
|
||||
log('files needed to recover state as of %s:', when)
|
||||
for f in needed:
|
||||
log('\t%s', f)
|
||||
else:
|
||||
log('no files found')
|
||||
return needed
|
||||
|
||||
# Scan the .dat file corresponding to the last full backup performed.
|
||||
# Return
|
||||
#
|
||||
# filename, startpos, endpos, checksum
|
||||
#
|
||||
# of the last incremental. If there is no .dat file, or the .dat file
|
||||
# is empty, return
|
||||
#
|
||||
# None, None, None, None
|
||||
|
||||
def scandat(repofiles):
|
||||
fullfile = repofiles[0]
|
||||
datfile = os.path.splitext(fullfile)[0] + '.dat'
|
||||
fn = startpos = endpos = sum = None # assume .dat file missing or empty
|
||||
try:
|
||||
fp = open(datfile)
|
||||
except IOError as e:
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
else:
|
||||
# We only care about the last one.
|
||||
lines = fp.readlines()
|
||||
fp.close()
|
||||
if lines:
|
||||
fn, startpos, endpos, sum = lines[-1].split()
|
||||
startpos = int(startpos)
|
||||
endpos = int(endpos)
|
||||
|
||||
return fn, startpos, endpos, sum
|
||||
|
||||
def delete_old_backups(options):
|
||||
# Delete all full backup files except for the most recent full backup file
|
||||
all = sorted(filter(is_data_file, os.listdir(options.repository)))
|
||||
|
||||
deletable = []
|
||||
full = []
|
||||
for fname in all:
|
||||
root, ext = os.path.splitext(fname)
|
||||
if ext in ('.fs', '.fsz'):
|
||||
full.append(fname)
|
||||
if ext in ('.fs', '.fsz', '.deltafs', '.deltafsz'):
|
||||
deletable.append(fname)
|
||||
|
||||
# keep most recent full
|
||||
if not full:
|
||||
return
|
||||
|
||||
recentfull = full.pop(-1)
|
||||
deletable.remove(recentfull)
|
||||
root, ext = os.path.splitext(recentfull)
|
||||
dat = root + '.dat'
|
||||
if dat in deletable:
|
||||
deletable.remove(dat)
|
||||
index = root + '.index'
|
||||
if index in deletable:
|
||||
deletable.remove(index)
|
||||
|
||||
for fname in deletable:
|
||||
log('removing old backup file %s (and .dat / .index)', fname)
|
||||
root, ext = os.path.splitext(fname)
|
||||
try:
|
||||
os.unlink(os.path.join(options.repository, root + '.dat'))
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(os.path.join(options.repository, root + '.index'))
|
||||
except OSError:
|
||||
pass
|
||||
os.unlink(os.path.join(options.repository, fname))
|
||||
|
||||
def do_full_backup(options):
|
||||
options.full = True
|
||||
tnow = gen_filedate(options)
|
||||
dest = os.path.join(options.repository, gen_filename(options, now=tnow))
|
||||
if os.path.exists(dest):
|
||||
raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
|
||||
# Find the file position of the last completed transaction.
|
||||
fs = FileStorage(options.file, read_only=True)
|
||||
# Note that the FileStorage ctor calls read_index() which scans the file
|
||||
# and returns "the position just after the last valid transaction record".
|
||||
# getSize() then returns this position, which is exactly what we want,
|
||||
# because we only want to copy stuff from the beginning of the file to the
|
||||
# last valid transaction record.
|
||||
pos = fs.getSize()
|
||||
# Save the storage index into the repository
|
||||
index_file = os.path.join(options.repository,
|
||||
gen_filename(options, '.index', tnow))
|
||||
log('writing index')
|
||||
fs._index.save(pos, index_file)
|
||||
fs.close()
|
||||
log('writing full backup: %s bytes to %s', pos, dest)
|
||||
sum = copyfile(options, dest, 0, pos)
|
||||
# Write the data file for this full backup
|
||||
datfile = os.path.splitext(dest)[0] + '.dat'
|
||||
fp = open(datfile, 'w')
|
||||
print(dest, 0, pos, sum, file=fp)
|
||||
fp.flush()
|
||||
os.fsync(fp.fileno())
|
||||
fp.close()
|
||||
if options.killold:
|
||||
delete_old_backups(options)
|
||||
|
||||
|
||||
def do_incremental_backup(options, reposz, repofiles):
|
||||
options.full = False
|
||||
tnow = gen_filedate(options)
|
||||
dest = os.path.join(options.repository, gen_filename(options, now=tnow))
|
||||
if os.path.exists(dest):
|
||||
raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
|
||||
# Find the file position of the last completed transaction.
|
||||
fs = FileStorage(options.file, read_only=True)
|
||||
# Note that the FileStorage ctor calls read_index() which scans the file
|
||||
# and returns "the position just after the last valid transaction record".
|
||||
# getSize() then returns this position, which is exactly what we want,
|
||||
# because we only want to copy stuff from the beginning of the file to the
|
||||
# last valid transaction record.
|
||||
pos = fs.getSize()
|
||||
log('writing index')
|
||||
index_file = os.path.join(options.repository,
|
||||
gen_filename(options, '.index', tnow))
|
||||
fs._index.save(pos, index_file)
|
||||
fs.close()
|
||||
log('writing incremental: %s bytes to %s', pos-reposz, dest)
|
||||
sum = copyfile(options, dest, reposz, pos - reposz)
|
||||
# The first file in repofiles points to the last full backup. Use this to
|
||||
# get the .dat file and append the information for this incrementatl to
|
||||
# that file.
|
||||
fullfile = repofiles[0]
|
||||
datfile = os.path.splitext(fullfile)[0] + '.dat'
|
||||
# This .dat file better exist. Let the exception percolate if not.
|
||||
fp = open(datfile, 'a')
|
||||
print(dest, reposz, pos, sum, file=fp)
|
||||
fp.flush()
|
||||
os.fsync(fp.fileno())
|
||||
fp.close()
|
||||
|
||||
|
||||
def do_backup(options):
|
||||
repofiles = find_files(options)
|
||||
# See if we need to do a full backup
|
||||
if options.full or not repofiles:
|
||||
log('doing a full backup')
|
||||
do_full_backup(options)
|
||||
return
|
||||
srcsz = os.path.getsize(options.file)
|
||||
if options.quick:
|
||||
fn, startpos, endpos, sum = scandat(repofiles)
|
||||
# If the .dat file was missing, or was empty, do a full backup
|
||||
if (fn, startpos, endpos, sum) == (None, None, None, None):
|
||||
log('missing or empty .dat file (full backup)')
|
||||
do_full_backup(options)
|
||||
return
|
||||
# Has the file shrunk, possibly because of a pack?
|
||||
if srcsz < endpos:
|
||||
log('file shrunk, possibly because of a pack (full backup)')
|
||||
do_full_backup(options)
|
||||
return
|
||||
# Now check the md5 sum of the source file, from the last
|
||||
# incremental's start and stop positions.
|
||||
srcfp = open(options.file, 'rb')
|
||||
srcfp.seek(startpos)
|
||||
srcsum = checksum(srcfp, endpos-startpos)
|
||||
srcfp.close()
|
||||
log('last incremental file: %s', fn)
|
||||
log('last incremental checksum: %s', sum)
|
||||
log('source checksum range: [%s..%s], sum: %s',
|
||||
startpos, endpos, srcsum)
|
||||
if sum == srcsum:
|
||||
if srcsz == endpos:
|
||||
log('No changes, nothing to do')
|
||||
return
|
||||
log('doing incremental, starting at: %s', endpos)
|
||||
do_incremental_backup(options, endpos, repofiles)
|
||||
return
|
||||
else:
|
||||
# This was is much slower, and more disk i/o intensive, but it's also
|
||||
# more accurate since it checks the actual existing files instead of
|
||||
# the information in the .dat file.
|
||||
#
|
||||
# See if we can do an incremental, based on the files that already
|
||||
# exist. This call of concat() will not write an output file.
|
||||
reposz, reposum = concat(repofiles)
|
||||
log('repository state: %s bytes, md5: %s', reposz, reposum)
|
||||
# Get the md5 checksum of the source file, up to two file positions:
|
||||
# the entire size of the file, and up to the file position of the last
|
||||
# incremental backup.
|
||||
srcfp = open(options.file, 'rb')
|
||||
srcsum = checksum(srcfp, srcsz)
|
||||
srcfp.seek(0)
|
||||
srcsum_backedup = checksum(srcfp, reposz)
|
||||
srcfp.close()
|
||||
log('current state : %s bytes, md5: %s', srcsz, srcsum)
|
||||
log('backed up state : %s bytes, md5: %s', reposz, srcsum_backedup)
|
||||
# Has nothing changed?
|
||||
if srcsz == reposz and srcsum == reposum:
|
||||
log('No changes, nothing to do')
|
||||
return
|
||||
# Has the file shrunk, probably because of a pack?
|
||||
if srcsz < reposz:
|
||||
log('file shrunk, possibly because of a pack (full backup)')
|
||||
do_full_backup(options)
|
||||
return
|
||||
# The source file is larger than the repository. If the md5 checksums
|
||||
# match, then we know we can do an incremental backup. If they don't,
|
||||
# then perhaps the file was packed at some point (or a
|
||||
# non-transactional undo was performed, but this is deprecated). Only
|
||||
# do a full backup if forced to.
|
||||
if reposum == srcsum_backedup:
|
||||
log('doing incremental, starting at: %s', reposz)
|
||||
do_incremental_backup(options, reposz, repofiles)
|
||||
return
|
||||
# The checksums don't match, meaning the front of the source file has
|
||||
# changed. We'll need to do a full backup in that case.
|
||||
log('file changed, possibly because of a pack (full backup)')
|
||||
do_full_backup(options)
|
||||
|
||||
|
||||
def do_recover(options):
|
||||
# Find the first full backup at or before the specified date
|
||||
repofiles = find_files(options)
|
||||
if not repofiles:
|
||||
if options.date:
|
||||
raise NoFiles('No files in repository before %s', options.date)
|
||||
else:
|
||||
raise NoFiles('No files in repository')
|
||||
if options.output is None:
|
||||
log('Recovering file to stdout')
|
||||
outfp = sys.stdout
|
||||
else:
|
||||
log('Recovering file to %s', options.output)
|
||||
outfp = open(options.output, 'wb')
|
||||
reposz, reposum = concat(repofiles, outfp)
|
||||
if outfp != sys.stdout:
|
||||
outfp.close()
|
||||
log('Recovered %s bytes, md5: %s', reposz, reposum)
|
||||
|
||||
if options.output is not None:
|
||||
last_base = os.path.splitext(repofiles[-1])[0]
|
||||
source_index = '%s.index' % last_base
|
||||
target_index = '%s.index' % options.output
|
||||
if os.path.exists(source_index):
|
||||
log('Restoring index file %s to %s', source_index, target_index)
|
||||
shutil.copyfile(source_index, target_index)
|
||||
else:
|
||||
log('No index file to restore: %s', source_index)
|
||||
|
||||
|
||||
def do_verify(options):
|
||||
# Verify the sizes and checksums of all files mentioned in the .dat file
|
||||
repofiles = find_files(options)
|
||||
if not repofiles:
|
||||
raise NoFiles('No files in repository')
|
||||
datfile = os.path.splitext(repofiles[0])[0] + '.dat'
|
||||
with open(datfile) as fp:
|
||||
for line in fp:
|
||||
fn, startpos, endpos, sum = line.split()
|
||||
startpos = int(startpos)
|
||||
endpos = int(endpos)
|
||||
filename = os.path.join(options.repository,
|
||||
os.path.basename(fn))
|
||||
expected_size = endpos - startpos
|
||||
log("Verifying %s", filename)
|
||||
try:
|
||||
if filename.endswith('fsz'):
|
||||
actual_sum, size = get_checksum_and_size_of_gzipped_file(filename, options.quick)
|
||||
when_uncompressed = ' (when uncompressed)'
|
||||
else:
|
||||
actual_sum, size = get_checksum_and_size_of_file(filename, options.quick)
|
||||
when_uncompressed = ''
|
||||
except IOError:
|
||||
error("%s is missing", filename)
|
||||
continue
|
||||
if size != expected_size:
|
||||
error("%s is %d bytes%s, should be %d bytes", filename,
|
||||
size, when_uncompressed, expected_size)
|
||||
elif not options.quick:
|
||||
if actual_sum != sum:
|
||||
error("%s has checksum %s%s instead of %s", filename,
|
||||
actual_sum, when_uncompressed, sum)
|
||||
|
||||
|
||||
def get_checksum_and_size_of_gzipped_file(filename, quick):
|
||||
with _GzipCloser(filename, 'rb') as fp:
|
||||
if quick:
|
||||
return None, file_size(fp)
|
||||
else:
|
||||
return checksum_and_size(fp)
|
||||
|
||||
|
||||
def get_checksum_and_size_of_file(filename, quick):
|
||||
with open(filename, 'rb') as fp:
|
||||
fp.seek(0, 2)
|
||||
actual_size = fp.tell()
|
||||
if quick:
|
||||
actual_sum = None
|
||||
else:
|
||||
fp.seek(0)
|
||||
actual_sum = checksum(fp, actual_size)
|
||||
return actual_sum, actual_size
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv[1:]
|
||||
options = parseargs(argv)
|
||||
if options.mode == BACKUP:
|
||||
try:
|
||||
do_backup(options)
|
||||
except WouldOverwriteFiles as e:
|
||||
sys.exit(str(e))
|
||||
elif options.mode == RECOVER:
|
||||
try:
|
||||
do_recover(options)
|
||||
except NoFiles as e:
|
||||
sys.exit(str(e))
|
||||
else:
|
||||
assert options.mode == VERIFY
|
||||
try:
|
||||
do_verify(options)
|
||||
except NoFiles as e:
|
||||
sys.exit(str(e))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python
|
||||
"""Report on the space used by objects in a storage.
|
||||
|
||||
usage: space.py data.fs
|
||||
|
||||
The current implementation only supports FileStorage.
|
||||
|
||||
Current limitations / simplifications: Ignores revisions and versions.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from ZODB.utils import U64, get_pickle_metadata, load_current
|
||||
import six
|
||||
|
||||
def run(path, v=0):
|
||||
fs = FileStorage(path, read_only=1)
|
||||
# break into the file implementation
|
||||
if hasattr(fs._index, 'iterkeys'):
|
||||
iter = six.iterkeys(fs._index)
|
||||
else:
|
||||
iter = fs._index.keys()
|
||||
totals = {}
|
||||
for oid in iter:
|
||||
data, serialno = load_current(fs, oid)
|
||||
mod, klass = get_pickle_metadata(data)
|
||||
key = "%s.%s" % (mod, klass)
|
||||
bytes, count = totals.get(key, (0, 0))
|
||||
bytes += len(data)
|
||||
count += 1
|
||||
totals[key] = bytes, count
|
||||
if v:
|
||||
print("%8s %5d %s" % (U64(oid), len(data), key))
|
||||
L = totals.items()
|
||||
L.sort(lambda a, b: cmp(a[1], b[1]))
|
||||
L.reverse()
|
||||
print("Totals per object class:")
|
||||
for key, (bytes, count) in L:
|
||||
print("%8d %8d %s" % (count, bytes, key))
|
||||
|
||||
def main():
|
||||
import sys
|
||||
import getopt
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "v")
|
||||
except getopt.error as msg:
|
||||
print(msg)
|
||||
print("usage: space.py [-v] Data.fs")
|
||||
sys.exit(2)
|
||||
if len(args) != 1:
|
||||
print("usage: space.py [-v] Data.fs")
|
||||
sys.exit(2)
|
||||
v = 0
|
||||
for o, a in opts:
|
||||
if o == "-v":
|
||||
v += 1
|
||||
path = args[0]
|
||||
run(path, v)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
====================
|
||||
The `fstail` utility
|
||||
====================
|
||||
|
||||
The `fstail` utility shows information for a FileStorage about the last `n`
|
||||
transactions:
|
||||
|
||||
We have to prepare a FileStorage first:
|
||||
|
||||
>>> from ZODB.FileStorage import FileStorage
|
||||
>>> from ZODB.DB import DB
|
||||
>>> import transaction
|
||||
>>> from tempfile import mktemp
|
||||
>>> storagefile = mktemp(suffix='.fs')
|
||||
>>> base_storage = FileStorage(storagefile)
|
||||
>>> database = DB(base_storage)
|
||||
>>> connection1 = database.open()
|
||||
>>> root = connection1.root()
|
||||
>>> root['foo'] = 1
|
||||
>>> transaction.commit()
|
||||
|
||||
Now lets have a look at the last transactions of this FileStorage:
|
||||
|
||||
>>> from ZODB.scripts.fstail import main
|
||||
>>> main(storagefile, 5)
|
||||
2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
|
||||
user='' description='' length=132 offset=162 (+23)
|
||||
<BLANKLINE>
|
||||
2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
|
||||
user='' description='initial database creation' length=150 offset=4 (+48)
|
||||
<BLANKLINE>
|
||||
|
||||
Now clean up the storage again:
|
||||
|
||||
>>> import os
|
||||
>>> connection1.close()
|
||||
>>> base_storage.close()
|
||||
>>> os.unlink(storagefile)
|
||||
>>> os.unlink(storagefile+'.index')
|
||||
>>> os.unlink(storagefile+'.lock')
|
||||
>>> os.unlink(storagefile+'.tmp')
|
||||
@@ -0,0 +1,43 @@
|
||||
Getting Object Referrers
|
||||
========================
|
||||
|
||||
The referrers module provides a way to get object referrers. It
|
||||
provides a referrers method that takes an iterable storage object. It
|
||||
returns a dictionary mapping object ids to lists of referrer object
|
||||
versions, which each version is a tuple an object id nd serial
|
||||
nummber.
|
||||
|
||||
To see how this works, we'll create a small database:
|
||||
|
||||
>>> import transaction
|
||||
>>> from persistent.mapping import PersistentMapping
|
||||
>>> from ZODB.FileStorage import FileStorage
|
||||
>>> from ZODB.DB import DB
|
||||
>>> import os, tempfile
|
||||
>>> dest = tempfile.mkdtemp()
|
||||
>>> fs = FileStorage(os.path.join(dest, 'Data.fs'))
|
||||
>>> db = DB(fs)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()['a'] = PersistentMapping()
|
||||
>>> conn.root()['b'] = PersistentMapping()
|
||||
>>> transaction.commit()
|
||||
>>> roid = conn.root()._p_oid
|
||||
>>> aoid = conn.root()['a']._p_oid
|
||||
>>> boid = conn.root()['b']._p_oid
|
||||
>>> s1 = conn.root()['b']._p_serial
|
||||
|
||||
>>> conn.root()['a']['b'] = conn.root()['b']
|
||||
>>> transaction.commit()
|
||||
>>> s2 = conn.root()['a']._p_serial
|
||||
|
||||
Now we'll get the storage and compute the referrers:
|
||||
|
||||
>>> import ZODB.scripts.referrers
|
||||
>>> referrers = ZODB.scripts.referrers.referrers(fs)
|
||||
|
||||
>>> referrers[boid] == [(roid, s1), (aoid, s2)]
|
||||
True
|
||||
|
||||
.. Cleanup
|
||||
|
||||
>>> db.close()
|
||||
@@ -0,0 +1,48 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2004 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
import doctest
|
||||
import re
|
||||
import unittest
|
||||
import ZODB.tests.util
|
||||
import zope.testing.renormalizing
|
||||
|
||||
checker = zope.testing.renormalizing.RENormalizing([
|
||||
(re.compile(
|
||||
r'[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+'),
|
||||
'2007-11-10 15:18:48.543001'),
|
||||
(re.compile('hash=[0-9a-f]{40}'),
|
||||
'hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3'),
|
||||
# Python 3 bytes add a "b".
|
||||
(re.compile("b('.*?')"), r"\1"),
|
||||
(re.compile('b(".*?")'), r"\1"),
|
||||
# Python 3 produces larger pickles, even when we use zodbpickle :(
|
||||
# this changes all the offsets and sizes in fstail.txt
|
||||
(re.compile("user='' description='' "
|
||||
r"length=[0-9]+ offset=[0-9]+ \(\+23\)"),
|
||||
"user='' description='' "
|
||||
"length=<LENGTH> offset=<OFFSET> (+23)"),
|
||||
(re.compile("user='' description='initial database creation' "
|
||||
r"length=[0-9]+ offset=4 \(\+48\)"),
|
||||
"user='' description='initial database creation' "
|
||||
"length=<LENGTH> offset=4 (+48)"),
|
||||
])
|
||||
|
||||
def test_suite():
|
||||
return unittest.TestSuite((
|
||||
doctest.DocFileSuite(
|
||||
'referrers.txt',
|
||||
'fstail.txt',
|
||||
setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown,
|
||||
checker=checker),
|
||||
))
|
||||
@@ -0,0 +1,55 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2010 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
import doctest
|
||||
import re
|
||||
import unittest
|
||||
|
||||
import ZODB
|
||||
from zope.testing import setupstack
|
||||
from zope.testing.renormalizing import RENormalizing
|
||||
|
||||
def test_fstest_verbose():
|
||||
r"""
|
||||
>>> db = ZODB.DB('data.fs')
|
||||
>>> db.close()
|
||||
>>> import ZODB.scripts.fstest
|
||||
>>> ZODB.scripts.fstest.main(['data.fs'])
|
||||
|
||||
>>> ZODB.scripts.fstest.main(['data.fs'])
|
||||
|
||||
>>> ZODB.scripts.fstest.main(['-v', 'data.fs'])
|
||||
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
||||
4: transaction tid ... #0
|
||||
no errors detected
|
||||
|
||||
>>> ZODB.scripts.fstest.main(['-vvv', 'data.fs'])
|
||||
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
||||
52: object oid 0x0000000000000000 #0
|
||||
4: transaction tid ... #0
|
||||
no errors detected
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def test_suite():
|
||||
checker = RENormalizing([
|
||||
# Python 3 drops the u'' prefix on unicode strings
|
||||
(re.compile(r"u('[^']*')"), r"\1"),
|
||||
])
|
||||
return unittest.TestSuite([
|
||||
doctest.DocTestSuite('ZODB.scripts.fstest', checker=checker),
|
||||
doctest.DocTestSuite(setUp=setupstack.setUpDirectory,
|
||||
tearDown=setupstack.tearDown),
|
||||
])
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,835 @@
|
||||
#!/usr/bin/env python
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Test script for testing ZODB under a heavy zope-like load.
|
||||
|
||||
Note that, to be as realistic as possible with ZEO, you should run this
|
||||
script multiple times, to simulate multiple clients.
|
||||
|
||||
Here's how this works.
|
||||
|
||||
The script starts some number of threads. Each thread, sequentially
|
||||
executes jobs. There is a job producer that produces jobs.
|
||||
|
||||
Input data are provided by a mail producer that hands out message from
|
||||
a mailbox.
|
||||
|
||||
Execution continues until there is an error, which will normally occur
|
||||
when the mailbox is exhausted.
|
||||
|
||||
Command-line options are used to provide job definitions. Job
|
||||
definitions have perameters of the form name=value. Jobs have 2
|
||||
standard parameters:
|
||||
|
||||
frequency=integer
|
||||
|
||||
The frequency of the job. The default is 1.
|
||||
|
||||
sleep=float
|
||||
|
||||
The number os seconds to sleep before performing the job. The
|
||||
default is 0.
|
||||
|
||||
Usage: loadmail2 [options]
|
||||
|
||||
Options:
|
||||
|
||||
-edit [frequency=integer] [sleep=float]
|
||||
|
||||
Define an edit job. An edit job edits a random already-saved
|
||||
email message, deleting and inserting a random number of words.
|
||||
|
||||
After editing the message, the message is (re)cataloged.
|
||||
|
||||
-insert [number=int] [frequency=integer] [sleep=float]
|
||||
|
||||
Insert some number of email messages.
|
||||
|
||||
-index [number=int] [frequency=integer] [sleep=float]
|
||||
|
||||
Insert and index (catalog) some number of email messages.
|
||||
|
||||
-search [terms='word1 word2 ...'] [frequency=integer] [sleep=float]
|
||||
|
||||
Search the catalog. A query is givem with one or more terms as
|
||||
would be entered into a typical seach box. If no query is
|
||||
given, then queries will be randomly selected based on a set of
|
||||
built-in word list.
|
||||
|
||||
-setup
|
||||
|
||||
Set up the database. This will delete any existing Data.fs
|
||||
file. (Of course, this may have no effect, if there is a
|
||||
custom_zodb that defined a different storage.) It also adds a
|
||||
mail folder and a catalog.
|
||||
|
||||
-options file
|
||||
|
||||
Read options from the given file. Th efile should be a python
|
||||
source file that defines a sequence of options named 'options'.
|
||||
|
||||
-threads n
|
||||
|
||||
Specify the number of threads to execute. If not specified (< 2),
|
||||
then jobs are run in a single (main) thread.
|
||||
|
||||
-mbox filename
|
||||
|
||||
Specify the mailbox for getting input data.
|
||||
|
||||
There is a (lame) syntax for providing options within the
|
||||
filename. The filename may be followed by up to 3 integers,
|
||||
min, max, and start:
|
||||
|
||||
-mbox 'foo.mbox 0 100 10000'
|
||||
|
||||
The messages from min to max will be read from the mailbox.
|
||||
They will be assigned message numbers starting with start.
|
||||
So, in the example above, we read the first hundred messages
|
||||
and assign thgem message numbers starting with 10001.
|
||||
|
||||
The maxmum can be given as a negative number, in which case, it
|
||||
specifies the number of messages to read.
|
||||
|
||||
The start defaults to the minimum. The following two options:
|
||||
|
||||
-mbox 'foo.mbox 300 400 300'
|
||||
|
||||
and
|
||||
|
||||
-mbox 'foo.mbox 300 -100'
|
||||
|
||||
are equivalent
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import mailbox
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import transaction
|
||||
|
||||
class JobProducer(object):
|
||||
|
||||
def __init__(self):
|
||||
self.jobs = []
|
||||
|
||||
def add(self, callable, frequency, sleep, repeatp=0):
|
||||
self.jobs.extend([(callable, sleep, repeatp)] * int(frequency))
|
||||
random.shuffle(self.jobs)
|
||||
|
||||
def next(self):
|
||||
factory, sleep, repeatp = random.choice(self.jobs)
|
||||
time.sleep(sleep)
|
||||
callable, args = factory.create()
|
||||
return factory, callable, args, repeatp
|
||||
|
||||
def __nonzero__(self):
|
||||
return not not self.jobs
|
||||
|
||||
|
||||
|
||||
class MBox(object):
|
||||
|
||||
def __init__(self, filename):
|
||||
if ' ' in filename:
|
||||
filename = filename.split()
|
||||
if len(filename) < 4:
|
||||
filename += [0, 0, -1][-(4-len(filename)):]
|
||||
filename, min, max, start = filename
|
||||
min = int(min)
|
||||
max = int(max)
|
||||
start = int(start)
|
||||
|
||||
if start < 0:
|
||||
start = min
|
||||
|
||||
if max < 0:
|
||||
# negative max is treated as a count
|
||||
self._max = start - max
|
||||
elif max > 0:
|
||||
self._max = start + max - min
|
||||
else:
|
||||
self._max = 0
|
||||
|
||||
else:
|
||||
self._max = 0
|
||||
min = start = 0
|
||||
|
||||
if filename.endswith('.bz2'):
|
||||
f = os.popen("bunzip2 <"+filename, 'r')
|
||||
filename = filename[-4:]
|
||||
else:
|
||||
f = open(filename)
|
||||
|
||||
self._mbox = mb = mailbox.UnixMailbox(f)
|
||||
|
||||
self.number = start
|
||||
while min:
|
||||
next(mb)
|
||||
min -= 1
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self.__name__ = os.path.splitext(os.path.split(filename)[1])[0]
|
||||
self._max = max
|
||||
|
||||
def next(self):
|
||||
with self.lock:
|
||||
if self._max > 0 and self.number >= self._max:
|
||||
raise IndexError(self.number + 1)
|
||||
message = next(self._mbox)
|
||||
message.body = message.fp.read()
|
||||
message.headers = list(message.headers)
|
||||
self.number += 1
|
||||
message.number = self.number
|
||||
message.mbox = self.__name__
|
||||
return message
|
||||
|
||||
bins = 9973
|
||||
#bins = 11
|
||||
def mailfolder(app, mboxname, number):
|
||||
mail = getattr(app, mboxname, None)
|
||||
if mail is None:
|
||||
app.manage_addFolder(mboxname)
|
||||
mail = getattr(app, mboxname)
|
||||
from BTrees.Length import Length
|
||||
mail.length = Length()
|
||||
for i in range(bins):
|
||||
mail.manage_addFolder('b'+str(i))
|
||||
bin = hash(str(number))%bins
|
||||
return getattr(mail, 'b'+str(bin))
|
||||
|
||||
|
||||
def VmSize():
|
||||
|
||||
try:
|
||||
with open('/proc/%s/status' % os.getpid()) as f:
|
||||
lines = f.readlines()
|
||||
except:
|
||||
return 0
|
||||
else:
|
||||
l = list(filter(lambda l: l[:7] == 'VmSize:', lines))
|
||||
if l:
|
||||
l = l[0][7:].strip().split()[0]
|
||||
return int(l)
|
||||
return 0
|
||||
|
||||
def setup(lib_python):
|
||||
try:
|
||||
os.remove(os.path.join(lib_python, '..', '..', 'var', 'Data.fs'))
|
||||
except:
|
||||
pass
|
||||
import Zope2
|
||||
import Products
|
||||
import AccessControl.SecurityManagement
|
||||
app=Zope2.app()
|
||||
|
||||
Products.ZCatalog.ZCatalog.manage_addZCatalog(app, 'cat', '')
|
||||
|
||||
from Products.ZCTextIndex.ZCTextIndex import PLexicon
|
||||
from Products.ZCTextIndex.Lexicon import Splitter, CaseNormalizer
|
||||
|
||||
app.cat._setObject('lex',
|
||||
PLexicon('lex', '', Splitter(), CaseNormalizer())
|
||||
)
|
||||
|
||||
class extra(object):
|
||||
doc_attr = 'PrincipiaSearchSource'
|
||||
lexicon_id = 'lex'
|
||||
index_type = 'Okapi BM25 Rank'
|
||||
|
||||
app.cat.addIndex('PrincipiaSearchSource', 'ZCTextIndex', extra)
|
||||
|
||||
transaction.commit()
|
||||
|
||||
system = AccessControl.SpecialUsers.system
|
||||
AccessControl.SecurityManagement.newSecurityManager(None, system)
|
||||
|
||||
app._p_jar.close()
|
||||
|
||||
def do(db, f, args):
|
||||
"""Do something in a transaction, retrying of necessary
|
||||
|
||||
Measure the speed of both the compurartion and the commit
|
||||
"""
|
||||
from ZODB.POSException import ConflictError
|
||||
wcomp = ccomp = wcommit = ccommit = 0.0
|
||||
rconflicts = wconflicts = 0
|
||||
start = time.time()
|
||||
|
||||
while 1:
|
||||
connection = db.open()
|
||||
try:
|
||||
transaction.begin()
|
||||
t=time.time()
|
||||
c=time.clock()
|
||||
try:
|
||||
try:
|
||||
r = f(connection, *args)
|
||||
except ConflictError:
|
||||
rconflicts += 1
|
||||
transaction.abort()
|
||||
continue
|
||||
finally:
|
||||
wcomp += time.time() - t
|
||||
ccomp += time.clock() - c
|
||||
|
||||
t=time.time()
|
||||
c=time.clock()
|
||||
try:
|
||||
try:
|
||||
transaction.commit()
|
||||
break
|
||||
except ConflictError:
|
||||
wconflicts += 1
|
||||
transaction.abort()
|
||||
continue
|
||||
finally:
|
||||
wcommit += time.time() - t
|
||||
ccommit += time.clock() - c
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return start, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit, r
|
||||
|
||||
def run1(tid, db, factory, job, args):
|
||||
(start, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit, r
|
||||
) = do(db, job, args)
|
||||
start = "%.4d-%.2d-%.2d %.2d:%.2d:%.2d" % time.localtime(start)[:6]
|
||||
print("%s %s %8.3g %8.3g %s %s\t%8.3g %8.3g %s %r" % (
|
||||
start, tid, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit,
|
||||
factory.__name__, r))
|
||||
|
||||
def run(jobs, tid=b''):
|
||||
import Zope2
|
||||
while 1:
|
||||
factory, job, args, repeatp = next(jobs)
|
||||
run1(tid, Zope2.DB, factory, job, args)
|
||||
if repeatp:
|
||||
while 1:
|
||||
i = random.randint(0,100)
|
||||
if i > repeatp:
|
||||
break
|
||||
run1(tid, Zope2.DB, factory, job, args)
|
||||
|
||||
|
||||
def index(connection, messages, catalog, max):
|
||||
app = connection.root()['Application']
|
||||
for message in messages:
|
||||
mail = mailfolder(app, message.mbox, message.number)
|
||||
|
||||
if max:
|
||||
# Cheat and use folder implementation secrets
|
||||
# to avoid having to read the old data
|
||||
_objects = mail._objects
|
||||
if len(_objects) >= max:
|
||||
for d in _objects[:len(_objects)-max+1]:
|
||||
del mail.__dict__[d['id']]
|
||||
mail._objects = _objects[len(_objects)-max+1:]
|
||||
|
||||
docid = 'm'+str(message.number)
|
||||
mail.manage_addDTMLDocument(docid, file=message.body)
|
||||
|
||||
# increment counted
|
||||
getattr(app, message.mbox).length.change(1)
|
||||
|
||||
doc = mail[docid]
|
||||
for h in message.headers:
|
||||
h = h.strip()
|
||||
l = h.find(':')
|
||||
if l <= 0:
|
||||
continue
|
||||
name = h[:l].lower()
|
||||
if name=='subject':
|
||||
name='title'
|
||||
v = h[l+1:].strip()
|
||||
type='string'
|
||||
|
||||
if name=='title':
|
||||
doc.manage_changeProperties(title=h)
|
||||
else:
|
||||
try:
|
||||
doc.manage_addProperty(name, v, type)
|
||||
except:
|
||||
pass
|
||||
if catalog:
|
||||
app.cat.catalog_object(doc)
|
||||
|
||||
return message.number
|
||||
|
||||
class IndexJob(object):
|
||||
needs_mbox = 1
|
||||
catalog = 1
|
||||
prefix = 'index'
|
||||
|
||||
def __init__(self, mbox, number=1, max=0):
|
||||
self.__name__ = "%s%s_%s" % (self.prefix, number, mbox.__name__)
|
||||
self.mbox, self.number, self.max = mbox, int(number), int(max)
|
||||
|
||||
def create(self):
|
||||
messages = [next(self.mbox) for i in range(self.number)]
|
||||
return index, (messages, self.catalog, self.max)
|
||||
|
||||
|
||||
class InsertJob(IndexJob):
|
||||
catalog = 0
|
||||
prefix = 'insert'
|
||||
|
||||
wordre = re.compile(r'(\w{3,20})')
|
||||
stop = 'and', 'not'
|
||||
def edit(connection, mbox, catalog=1):
|
||||
app = connection.root()['Application']
|
||||
mail = getattr(app, mbox.__name__, None)
|
||||
if mail is None:
|
||||
time.sleep(1)
|
||||
return "No mailbox %s" % mbox.__name__
|
||||
|
||||
nmessages = mail.length()
|
||||
if nmessages < 2:
|
||||
time.sleep(1)
|
||||
return "No messages to edit in %s" % mbox.__name__
|
||||
|
||||
# find a message to edit:
|
||||
while 1:
|
||||
number = random.randint(1, nmessages-1)
|
||||
did = 'm' + str(number)
|
||||
|
||||
mail = mailfolder(app, mbox.__name__, number)
|
||||
doc = getattr(mail, did, None)
|
||||
if doc is not None:
|
||||
break
|
||||
|
||||
text = doc.raw.split()
|
||||
norig = len(text)
|
||||
if norig > 10:
|
||||
ndel = int(math.exp(random.randint(0, int(math.log(norig)))))
|
||||
nins = int(math.exp(random.randint(0, int(math.log(norig)))))
|
||||
else:
|
||||
ndel = 0
|
||||
nins = 10
|
||||
|
||||
for j in range(ndel):
|
||||
j = random.randint(0,len(text)-1)
|
||||
word = text[j]
|
||||
m = wordre.search(word)
|
||||
if m:
|
||||
word = m.group(1).lower()
|
||||
if (word not in wordsd) and word not in stop:
|
||||
words.append(word)
|
||||
wordsd[word] = 1
|
||||
del text[j]
|
||||
|
||||
for j in range(nins):
|
||||
word = random.choice(words)
|
||||
text.append(word)
|
||||
|
||||
doc.raw = ' '.join(text)
|
||||
|
||||
if catalog:
|
||||
app.cat.catalog_object(doc)
|
||||
|
||||
return norig, ndel, nins
|
||||
|
||||
class EditJob(object):
|
||||
needs_mbox = 1
|
||||
prefix = 'edit'
|
||||
catalog = 1
|
||||
|
||||
def __init__(self, mbox):
|
||||
self.__name__ = "%s_%s" % (self.prefix, mbox.__name__)
|
||||
self.mbox = mbox
|
||||
|
||||
def create(self):
|
||||
return edit, (self.mbox, self.catalog)
|
||||
|
||||
class ModifyJob(EditJob):
|
||||
prefix = 'modify'
|
||||
catalog = 0
|
||||
|
||||
|
||||
def search(connection, terms, number):
|
||||
app = connection.root()['Application']
|
||||
cat = app.cat
|
||||
n = 0
|
||||
|
||||
for i in number:
|
||||
term = random.choice(terms)
|
||||
|
||||
results = cat(PrincipiaSearchSource=term)
|
||||
n += len(results)
|
||||
for result in results:
|
||||
obj = result.getObject()
|
||||
# Apparently, there is a bug in Zope that leads obj to be None
|
||||
# on occasion.
|
||||
if obj is not None:
|
||||
obj.getId()
|
||||
|
||||
return n
|
||||
|
||||
class SearchJob(object):
|
||||
|
||||
def __init__(self, terms='', number=10):
|
||||
|
||||
if terms:
|
||||
terms = terms.split()
|
||||
self.__name__ = "search_" + '_'.join(terms)
|
||||
self.terms = terms
|
||||
else:
|
||||
self.__name__ = 'search'
|
||||
self.terms = words
|
||||
|
||||
number = min(int(number), len(self.terms))
|
||||
self.number = list(range(number))
|
||||
|
||||
def create(self):
|
||||
return search, (self.terms, self.number)
|
||||
|
||||
|
||||
words=['banishment', 'indirectly', 'imprecise', 'peeks',
|
||||
'opportunely', 'bribe', 'sufficiently', 'Occidentalized', 'elapsing',
|
||||
'fermenting', 'listen', 'orphanage', 'younger', 'draperies', 'Ida',
|
||||
'cuttlefish', 'mastermind', 'Michaels', 'populations', 'lent',
|
||||
'cater', 'attentional', 'hastiness', 'dragnet', 'mangling',
|
||||
'scabbards', 'princely', 'star', 'repeat', 'deviation', 'agers',
|
||||
'fix', 'digital', 'ambitious', 'transit', 'jeeps', 'lighted',
|
||||
'Prussianizations', 'Kickapoo', 'virtual', 'Andrew', 'generally',
|
||||
'boatsman', 'amounts', 'promulgation', 'Malay', 'savaging',
|
||||
'courtesan', 'nursed', 'hungered', 'shiningly', 'ship', 'presides',
|
||||
'Parke', 'moderns', 'Jonas', 'unenlightening', 'dearth', 'deer',
|
||||
'domesticates', 'recognize', 'gong', 'penetrating', 'dependents',
|
||||
'unusually', 'complications', 'Dennis', 'imbalances', 'nightgown',
|
||||
'attached', 'testaments', 'congresswoman', 'circuits', 'bumpers',
|
||||
'braver', 'Boreas', 'hauled', 'Howe', 'seethed', 'cult', 'numismatic',
|
||||
'vitality', 'differences', 'collapsed', 'Sandburg', 'inches', 'head',
|
||||
'rhythmic', 'opponent', 'blanketer', 'attorneys', 'hen', 'spies',
|
||||
'indispensably', 'clinical', 'redirection', 'submit', 'catalysts',
|
||||
'councilwoman', 'kills', 'topologies', 'noxious', 'exactions',
|
||||
'dashers', 'balanced', 'slider', 'cancerous', 'bathtubs', 'legged',
|
||||
'respectably', 'crochets', 'absenteeism', 'arcsine', 'facility',
|
||||
'cleaners', 'bobwhite', 'Hawkins', 'stockade', 'provisional',
|
||||
'tenants', 'forearms', 'Knowlton', 'commit', 'scornful',
|
||||
'pediatrician', 'greets', 'clenches', 'trowels', 'accepts',
|
||||
'Carboloy', 'Glenn', 'Leigh', 'enroll', 'Madison', 'Macon', 'oiling',
|
||||
'entertainingly', 'super', 'propositional', 'pliers', 'beneficiary',
|
||||
'hospitable', 'emigration', 'sift', 'sensor', 'reserved',
|
||||
'colonization', 'shrilled', 'momentously', 'stevedore', 'Shanghaiing',
|
||||
'schoolmasters', 'shaken', 'biology', 'inclination', 'immoderate',
|
||||
'stem', 'allegory', 'economical', 'daytime', 'Newell', 'Moscow',
|
||||
'archeology', 'ported', 'scandals', 'Blackfoot', 'leery', 'kilobit',
|
||||
'empire', 'obliviousness', 'productions', 'sacrificed', 'ideals',
|
||||
'enrolling', 'certainties', 'Capsicum', 'Brookdale', 'Markism',
|
||||
'unkind', 'dyers', 'legislates', 'grotesquely', 'megawords',
|
||||
'arbitrary', 'laughing', 'wildcats', 'thrower', 'sex', 'devils',
|
||||
'Wehr', 'ablates', 'consume', 'gossips', 'doorways', 'Shari',
|
||||
'advanced', 'enumerable', 'existentially', 'stunt', 'auctioneers',
|
||||
'scheduler', 'blanching', 'petulance', 'perceptibly', 'vapors',
|
||||
'progressed', 'rains', 'intercom', 'emergency', 'increased',
|
||||
'fluctuating', 'Krishna', 'silken', 'reformed', 'transformation',
|
||||
'easter', 'fares', 'comprehensible', 'trespasses', 'hallmark',
|
||||
'tormenter', 'breastworks', 'brassiere', 'bladders', 'civet', 'death',
|
||||
'transformer', 'tolerably', 'bugle', 'clergy', 'mantels', 'satin',
|
||||
'Boswellizes', 'Bloomington', 'notifier', 'Filippo', 'circling',
|
||||
'unassigned', 'dumbness', 'sentries', 'representativeness', 'souped',
|
||||
'Klux', 'Kingstown', 'gerund', 'Russell', 'splices', 'bellow',
|
||||
'bandies', 'beefers', 'cameramen', 'appalled', 'Ionian', 'butterball',
|
||||
'Portland', 'pleaded', 'admiringly', 'pricks', 'hearty', 'corer',
|
||||
'deliverable', 'accountably', 'mentors', 'accorded',
|
||||
'acknowledgement', 'Lawrenceville', 'morphology', 'eucalyptus',
|
||||
'Rena', 'enchanting', 'tighter', 'scholars', 'graduations', 'edges',
|
||||
'Latinization', 'proficiency', 'monolithic', 'parenthesizing', 'defy',
|
||||
'shames', 'enjoyment', 'Purdue', 'disagrees', 'barefoot', 'maims',
|
||||
'flabbergast', 'dishonorable', 'interpolation', 'fanatics', 'dickens',
|
||||
'abysses', 'adverse', 'components', 'bowl', 'belong', 'Pipestone',
|
||||
'trainees', 'paw', 'pigtail', 'feed', 'whore', 'conditioner',
|
||||
'Volstead', 'voices', 'strain', 'inhabits', 'Edwin', 'discourses',
|
||||
'deigns', 'cruiser', 'biconvex', 'biking', 'depreciation', 'Harrison',
|
||||
'Persian', 'stunning', 'agar', 'rope', 'wagoner', 'elections',
|
||||
'reticulately', 'Cruz', 'pulpits', 'wilt', 'peels', 'plants',
|
||||
'administerings', 'deepen', 'rubs', 'hence', 'dissension', 'implored',
|
||||
'bereavement', 'abyss', 'Pennsylvania', 'benevolent', 'corresponding',
|
||||
'Poseidon', 'inactive', 'butchers', 'Mach', 'woke', 'loading',
|
||||
'utilizing', 'Hoosier', 'undo', 'Semitization', 'trigger', 'Mouthe',
|
||||
'mark', 'disgracefully', 'copier', 'futility', 'gondola', 'algebraic',
|
||||
'lecturers', 'sponged', 'instigators', 'looted', 'ether', 'trust',
|
||||
'feeblest', 'sequencer', 'disjointness', 'congresses', 'Vicksburg',
|
||||
'incompatibilities', 'commend', 'Luxembourg', 'reticulation',
|
||||
'instructively', 'reconstructs', 'bricks', 'attache', 'Englishman',
|
||||
'provocation', 'roughen', 'cynic', 'plugged', 'scrawls', 'antipode',
|
||||
'injected', 'Daedalus', 'Burnsides', 'asker', 'confronter',
|
||||
'merriment', 'disdain', 'thicket', 'stinker', 'great', 'tiers',
|
||||
'oust', 'antipodes', 'Macintosh', 'tented', 'packages',
|
||||
'Mediterraneanize', 'hurts', 'orthodontist', 'seeder', 'readying',
|
||||
'babying', 'Florida', 'Sri', 'buckets', 'complementary',
|
||||
'cartographer', 'chateaus', 'shaves', 'thinkable', 'Tehran',
|
||||
'Gordian', 'Angles', 'arguable', 'bureau', 'smallest', 'fans',
|
||||
'navigated', 'dipole', 'bootleg', 'distinctive', 'minimization',
|
||||
'absorbed', 'surmised', 'Malawi', 'absorbent', 'close', 'conciseness',
|
||||
'hopefully', 'declares', 'descent', 'trick', 'portend', 'unable',
|
||||
'mildly', 'Morse', 'reference', 'scours', 'Caribbean', 'battlers',
|
||||
'astringency', 'likelier', 'Byronizes', 'econometric', 'grad',
|
||||
'steak', 'Austrian', 'ban', 'voting', 'Darlington', 'bison', 'Cetus',
|
||||
'proclaim', 'Gilbertson', 'evictions', 'submittal', 'bearings',
|
||||
'Gothicizer', 'settings', 'McMahon', 'densities', 'determinants',
|
||||
'period', 'DeKastere', 'swindle', 'promptness', 'enablers', 'wordy',
|
||||
'during', 'tables', 'responder', 'baffle', 'phosgene', 'muttering',
|
||||
'limiters', 'custodian', 'prevented', 'Stouffer', 'waltz', 'Videotex',
|
||||
'brainstorms', 'alcoholism', 'jab', 'shouldering', 'screening',
|
||||
'explicitly', 'earner', 'commandment', 'French', 'scrutinizing',
|
||||
'Gemma', 'capacitive', 'sheriff', 'herbivore', 'Betsey', 'Formosa',
|
||||
'scorcher', 'font', 'damming', 'soldiers', 'flack', 'Marks',
|
||||
'unlinking', 'serenely', 'rotating', 'converge', 'celebrities',
|
||||
'unassailable', 'bawling', 'wording', 'silencing', 'scotch',
|
||||
'coincided', 'masochists', 'graphs', 'pernicious', 'disease',
|
||||
'depreciates', 'later', 'torus', 'interject', 'mutated', 'causer',
|
||||
'messy', 'Bechtel', 'redundantly', 'profoundest', 'autopsy',
|
||||
'philosophic', 'iterate', 'Poisson', 'horridly', 'silversmith',
|
||||
'millennium', 'plunder', 'salmon', 'missioner', 'advances', 'provers',
|
||||
'earthliness', 'manor', 'resurrectors', 'Dahl', 'canto', 'gangrene',
|
||||
'gabler', 'ashore', 'frictionless', 'expansionism', 'emphasis',
|
||||
'preservations', 'Duane', 'descend', 'isolated', 'firmware',
|
||||
'dynamites', 'scrawled', 'cavemen', 'ponder', 'prosperity', 'squaw',
|
||||
'vulnerable', 'opthalmic', 'Simms', 'unite', 'totallers', 'Waring',
|
||||
'enforced', 'bridge', 'collecting', 'sublime', 'Moore', 'gobble',
|
||||
'criticizes', 'daydreams', 'sedate', 'apples', 'Concordia',
|
||||
'subsequence', 'distill', 'Allan', 'seizure', 'Isadore', 'Lancashire',
|
||||
'spacings', 'corresponded', 'hobble', 'Boonton', 'genuineness',
|
||||
'artifact', 'gratuities', 'interviewee', 'Vladimir', 'mailable',
|
||||
'Bini', 'Kowalewski', 'interprets', 'bereave', 'evacuated', 'friend',
|
||||
'tourists', 'crunched', 'soothsayer', 'fleetly', 'Romanizations',
|
||||
'Medicaid', 'persevering', 'flimsy', 'doomsday', 'trillion',
|
||||
'carcasses', 'guess', 'seersucker', 'ripping', 'affliction',
|
||||
'wildest', 'spokes', 'sheaths', 'procreate', 'rusticates', 'Schapiro',
|
||||
'thereafter', 'mistakenly', 'shelf', 'ruination', 'bushel',
|
||||
'assuredly', 'corrupting', 'federation', 'portmanteau', 'wading',
|
||||
'incendiary', 'thing', 'wanderers', 'messages', 'Paso', 'reexamined',
|
||||
'freeings', 'denture', 'potting', 'disturber', 'laborer', 'comrade',
|
||||
'intercommunicating', 'Pelham', 'reproach', 'Fenton', 'Alva', 'oasis',
|
||||
'attending', 'cockpit', 'scout', 'Jude', 'gagging', 'jailed',
|
||||
'crustaceans', 'dirt', 'exquisitely', 'Internet', 'blocker', 'smock',
|
||||
'Troutman', 'neighboring', 'surprise', 'midscale', 'impart',
|
||||
'badgering', 'fountain', 'Essen', 'societies', 'redresses',
|
||||
'afterwards', 'puckering', 'silks', 'Blakey', 'sequel', 'greet',
|
||||
'basements', 'Aubrey', 'helmsman', 'album', 'wheelers', 'easternmost',
|
||||
'flock', 'ambassadors', 'astatine', 'supplant', 'gird', 'clockwork',
|
||||
'foxes', 'rerouting', 'divisional', 'bends', 'spacer',
|
||||
'physiologically', 'exquisite', 'concerts', 'unbridled', 'crossing',
|
||||
'rock', 'leatherneck', 'Fortescue', 'reloading', 'Laramie', 'Tim',
|
||||
'forlorn', 'revert', 'scarcer', 'spigot', 'equality', 'paranormal',
|
||||
'aggrieves', 'pegs', 'committeewomen', 'documented', 'interrupt',
|
||||
'emerald', 'Battelle', 'reconverted', 'anticipated', 'prejudices',
|
||||
'drowsiness', 'trivialities', 'food', 'blackberries', 'Cyclades',
|
||||
'tourist', 'branching', 'nugget', 'Asilomar', 'repairmen', 'Cowan',
|
||||
'receptacles', 'nobler', 'Nebraskan', 'territorial', 'chickadee',
|
||||
'bedbug', 'darted', 'vigilance', 'Octavia', 'summands', 'policemen',
|
||||
'twirls', 'style', 'outlawing', 'specifiable', 'pang', 'Orpheus',
|
||||
'epigram', 'Babel', 'butyrate', 'wishing', 'fiendish', 'accentuate',
|
||||
'much', 'pulsed', 'adorned', 'arbiters', 'counted', 'Afrikaner',
|
||||
'parameterizes', 'agenda', 'Americanism', 'referenda', 'derived',
|
||||
'liquidity', 'trembling', 'lordly', 'Agway', 'Dillon', 'propellers',
|
||||
'statement', 'stickiest', 'thankfully', 'autograph', 'parallel',
|
||||
'impulse', 'Hamey', 'stylistic', 'disproved', 'inquirer', 'hoisting',
|
||||
'residues', 'variant', 'colonials', 'dequeued', 'especial', 'Samoa',
|
||||
'Polaris', 'dismisses', 'surpasses', 'prognosis', 'urinates',
|
||||
'leaguers', 'ostriches', 'calculative', 'digested', 'divided',
|
||||
'reconfigurer', 'Lakewood', 'illegalities', 'redundancy',
|
||||
'approachability', 'masterly', 'cookery', 'crystallized', 'Dunham',
|
||||
'exclaims', 'mainline', 'Australianizes', 'nationhood', 'pusher',
|
||||
'ushers', 'paranoia', 'workstations', 'radiance', 'impedes',
|
||||
'Minotaur', 'cataloging', 'bites', 'fashioning', 'Alsop', 'servants',
|
||||
'Onondaga', 'paragraph', 'leadings', 'clients', 'Latrobe',
|
||||
'Cornwallis', 'excitingly', 'calorimetric', 'savior', 'tandem',
|
||||
'antibiotics', 'excuse', 'brushy', 'selfish', 'naive', 'becomes',
|
||||
'towers', 'popularizes', 'engender', 'introducing', 'possession',
|
||||
'slaughtered', 'marginally', 'Packards', 'parabola', 'utopia',
|
||||
'automata', 'deterrent', 'chocolates', 'objectives', 'clannish',
|
||||
'aspirin', 'ferociousness', 'primarily', 'armpit', 'handfuls',
|
||||
'dangle', 'Manila', 'enlivened', 'decrease', 'phylum', 'hardy',
|
||||
'objectively', 'baskets', 'chaired', 'Sepoy', 'deputy', 'blizzard',
|
||||
'shootings', 'breathtaking', 'sticking', 'initials', 'epitomized',
|
||||
'Forrest', 'cellular', 'amatory', 'radioed', 'horrified', 'Neva',
|
||||
'simultaneous', 'delimiter', 'expulsion', 'Himmler', 'contradiction',
|
||||
'Remus', 'Franklinizations', 'luggage', 'moisture', 'Jews',
|
||||
'comptroller', 'brevity', 'contradictions', 'Ohio', 'active',
|
||||
'babysit', 'China', 'youngest', 'superstition', 'clawing', 'raccoons',
|
||||
'chose', 'shoreline', 'helmets', 'Jeffersonian', 'papered',
|
||||
'kindergarten', 'reply', 'succinct', 'split', 'wriggle', 'suitcases',
|
||||
'nonce', 'grinders', 'anthem', 'showcase', 'maimed', 'blue', 'obeys',
|
||||
'unreported', 'perusing', 'recalculate', 'rancher', 'demonic',
|
||||
'Lilliputianize', 'approximation', 'repents', 'yellowness',
|
||||
'irritates', 'Ferber', 'flashlights', 'booty', 'Neanderthal',
|
||||
'someday', 'foregoes', 'lingering', 'cloudiness', 'guy', 'consumer',
|
||||
'Berkowitz', 'relics', 'interpolating', 'reappearing', 'advisements',
|
||||
'Nolan', 'turrets', 'skeletal', 'skills', 'mammas', 'Winsett',
|
||||
'wheelings', 'stiffen', 'monkeys', 'plainness', 'braziers', 'Leary',
|
||||
'advisee', 'jack', 'verb', 'reinterpret', 'geometrical', 'trolleys',
|
||||
'arboreal', 'overpowered', 'Cuzco', 'poetical', 'admirations',
|
||||
'Hobbes', 'phonemes', 'Newsweek', 'agitator', 'finally', 'prophets',
|
||||
'environment', 'easterners', 'precomputed', 'faults', 'rankly',
|
||||
'swallowing', 'crawl', 'trolley', 'spreading', 'resourceful', 'go',
|
||||
'demandingly', 'broader', 'spiders', 'Marsha', 'debris', 'operates',
|
||||
'Dundee', 'alleles', 'crunchier', 'quizzical', 'hanging', 'Fisk']
|
||||
|
||||
wordsd = {}
|
||||
for word in words:
|
||||
wordsd[word] = 1
|
||||
|
||||
|
||||
def collect_options(args, jobs, options):
|
||||
|
||||
while args:
|
||||
arg = args.pop(0)
|
||||
if arg.startswith('-'):
|
||||
name = arg[1:]
|
||||
if name == 'options':
|
||||
fname = args.pop(0)
|
||||
d = {}
|
||||
with open(fname) as fp:
|
||||
exec(compile(fp.read(), fname, 'exec'), d)
|
||||
collect_options(list(d['options']), jobs, options)
|
||||
elif name in options:
|
||||
v = args.pop(0)
|
||||
if options[name] != None:
|
||||
raise ValueError(
|
||||
"Duplicate values for %s, %s and %s"
|
||||
% (name, v, options[name])
|
||||
)
|
||||
options[name] = v
|
||||
elif name == 'setup':
|
||||
options['setup'] = 1
|
||||
elif name.capitalize()+'Job' in globals():
|
||||
job = name
|
||||
kw = {}
|
||||
while args and args[0].find("=") > 0:
|
||||
arg = args.pop(0).split('=')
|
||||
name, v = arg[0], '='.join(arg[1:])
|
||||
if name in kw:
|
||||
raise ValueError(
|
||||
"Duplicate parameter %s for job %s"
|
||||
% (name, job)
|
||||
)
|
||||
kw[name]=v
|
||||
if 'frequency' in kw:
|
||||
frequency = kw['frequency']
|
||||
del kw['frequency']
|
||||
else:
|
||||
frequency = 1
|
||||
|
||||
if 'sleep' in kw:
|
||||
sleep = float(kw['sleep'])
|
||||
del kw['sleep']
|
||||
else:
|
||||
sleep = 0.0001
|
||||
|
||||
if 'repeat' in kw:
|
||||
repeatp = float(kw['repeat'])
|
||||
del kw['repeat']
|
||||
else:
|
||||
repeatp = 0
|
||||
|
||||
jobs.append((job, kw, frequency, sleep, repeatp))
|
||||
else:
|
||||
raise ValueError("not an option or job", name)
|
||||
else:
|
||||
raise ValueError("Expected an option", arg)
|
||||
|
||||
|
||||
def find_lib_python():
|
||||
for b in os.getcwd(), os.path.split(sys.argv[0])[0]:
|
||||
for i in range(6):
|
||||
d = ['..']*i + ['lib', 'python']
|
||||
p = os.path.join(b, *d)
|
||||
if os.path.isdir(p):
|
||||
return p
|
||||
raise ValueError("Couldn't find lib/python")
|
||||
|
||||
def main(args=None):
|
||||
lib_python = find_lib_python()
|
||||
sys.path.insert(0, lib_python)
|
||||
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
if not args:
|
||||
print(__doc__)
|
||||
sys.exit(0)
|
||||
|
||||
print(args)
|
||||
random.seed(hash(tuple(args))) # always use the same for the given args
|
||||
|
||||
options = {"mbox": None, "threads": None}
|
||||
jobdefs = []
|
||||
collect_options(args, jobdefs, options)
|
||||
|
||||
mboxes = {}
|
||||
if options["mbox"]:
|
||||
mboxes[options["mbox"]] = MBox(options["mbox"])
|
||||
|
||||
# Perform a ZConfig-based Zope initialization:
|
||||
zetup(os.path.join(lib_python, '..', '..', 'etc', 'zope.conf'))
|
||||
|
||||
if 'setup' in options:
|
||||
setup(lib_python)
|
||||
else:
|
||||
import Zope2
|
||||
Zope2.startup()
|
||||
|
||||
jobs = JobProducer()
|
||||
for job, kw, frequency, sleep, repeatp in jobdefs:
|
||||
Job = globals()[job.capitalize()+'Job']
|
||||
if getattr(Job, 'needs_mbox', 0):
|
||||
if "mbox" not in kw:
|
||||
if not options["mbox"]:
|
||||
raise ValueError(
|
||||
"no mailbox (mbox option) file specified")
|
||||
kw['mbox'] = mboxes[options["mbox"]]
|
||||
else:
|
||||
if not mboxes.has_key[kw["mbox"]]:
|
||||
mboxes[kw['mbox']] = MBox[kw['mbox']]
|
||||
kw["mbox"] = mboxes[kw['mbox']]
|
||||
jobs.add(Job(**kw), frequency, sleep, repeatp)
|
||||
|
||||
if not jobs:
|
||||
print("No jobs to execute")
|
||||
return
|
||||
|
||||
threads = int(options['threads'] or '0')
|
||||
if threads > 1:
|
||||
threads = [threading.Thread(target=run, args=(jobs, i), name=str(i))
|
||||
for i in range(threads)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
else:
|
||||
run(jobs)
|
||||
|
||||
|
||||
def zetup(configfile_name):
|
||||
from Zope.Startup.options import ZopeOptions
|
||||
from Zope.Startup import handlers as h
|
||||
from App import config
|
||||
opts = ZopeOptions()
|
||||
opts.configfile = configfile_name
|
||||
opts.realize(args=[])
|
||||
h.handleConfig(opts.configroot, opts.confighandlers)
|
||||
config.setConfiguration(opts.configroot)
|
||||
from Zope.Startup import dropPrivileges
|
||||
dropPrivileges(opts.configroot)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,714 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2003 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Support for ZODB object serialization.
|
||||
|
||||
ZODB serializes objects using a custom format based on Python pickles.
|
||||
When an object is unserialized, it can be loaded as either a ghost or
|
||||
a real object. A ghost is a persistent object of the appropriate type
|
||||
but without any state. The first time a ghost is accessed, the
|
||||
persistence machinery traps access and loads the actual state. A
|
||||
ghost allows many persistent objects to be loaded while minimizing the
|
||||
memory consumption of referenced but otherwise unused objects.
|
||||
|
||||
Pickle format
|
||||
-------------
|
||||
|
||||
ZODB stores serialized objects using a custom format based on pickle.
|
||||
Each serialized object has two parts: the class description and the
|
||||
object state. The class description must provide enough information
|
||||
to call the class's ``__new__`` and create an empty object. Once the
|
||||
object exists as a ghost, its state is passed to ``__setstate__``.
|
||||
|
||||
The class description can be in a variety of formats, in part to
|
||||
provide backwards compatibility with earlier versions of Zope. The
|
||||
four current formats for class description are:
|
||||
|
||||
1. type(obj)
|
||||
2. type(obj), obj.__getnewargs__()
|
||||
3. (module name, class name), None
|
||||
7. (module name, class name), obj.__getnewargs__()
|
||||
|
||||
The second of these options is used if the object has a __getnewargs__()
|
||||
method. It is intended to support objects like persistent classes that have
|
||||
custom C layouts that are determined by arguments to __new__(). The
|
||||
third and fourth (#3 & #7) apply to instances of a persistent class (which
|
||||
means the class itself is persistent, not that it's a subclass of
|
||||
Persistent).
|
||||
|
||||
The type object is usually stored using the standard pickle mechanism, which
|
||||
involves the pickle GLOBAL opcode (giving the type's module and name as
|
||||
strings). The type may itself be a persistent object, in which case a
|
||||
persistent reference (see below) is used.
|
||||
|
||||
It's unclear what "usually" means in the last paragraph. There are two
|
||||
useful places to concentrate confusion about exactly which formats exist:
|
||||
|
||||
- ObjectReader.getClassName() below returns a dotted "module.class"
|
||||
string, via actually loading a pickle. This requires that the
|
||||
implementation of application objects be available.
|
||||
|
||||
- ZODB/utils.py's get_pickle_metadata() tries to return the module and
|
||||
class names (as strings) without importing any application modules or
|
||||
classes, via analyzing the pickle.
|
||||
|
||||
Earlier versions of Zope supported several other kinds of class
|
||||
descriptions. The current serialization code reads these descriptions, but
|
||||
does not write them. The three earlier formats are:
|
||||
|
||||
4. (module name, class name), __getinitargs__()
|
||||
5. class, None
|
||||
6. class, __getinitargs__()
|
||||
|
||||
Formats 4 and 6 are used only if the class defines a __getinitargs__()
|
||||
method, but we really can't tell them apart from formats 7 and 2
|
||||
(respectively). Formats 5 and 6 are used if the class does not have a
|
||||
__module__ attribute (I'm not sure when this applies, but I think it occurs
|
||||
for some but not all ZClasses).
|
||||
|
||||
|
||||
Persistent references
|
||||
---------------------
|
||||
|
||||
When one persistent object pickle refers to another persistent object,
|
||||
the database uses a persistent reference.
|
||||
|
||||
ZODB persistent references are of the form::
|
||||
|
||||
oid
|
||||
A simple object reference.
|
||||
|
||||
(oid, class meta data)
|
||||
A persistent object reference
|
||||
|
||||
[reference_type, args]
|
||||
An extended reference
|
||||
|
||||
Extension references come in a number of subforms, based on the
|
||||
reference types.
|
||||
|
||||
The following reference types are defined:
|
||||
|
||||
'w'
|
||||
Persistent weak reference. The arguments consist of an oid
|
||||
and optionally a database name.
|
||||
|
||||
The following are planned for the future:
|
||||
|
||||
'n'
|
||||
Multi-database simple object reference. The arguments consist
|
||||
of a database name, and an object id.
|
||||
|
||||
'm'
|
||||
Multi-database persistent object reference. The arguments consist
|
||||
of a database name, an object id, and class meta data.
|
||||
|
||||
The following legacy format is also supported.
|
||||
|
||||
[oid]
|
||||
A persistent weak reference
|
||||
|
||||
Because the persistent object reference forms include class
|
||||
information, it is not possible to change the class of a persistent
|
||||
object for which this form is used. If a transaction changed the
|
||||
class of an object, a new record with new class metadata would be
|
||||
written but all the old references would still use the old class. (It
|
||||
is possible that we could deal with this limitation in the future.)
|
||||
|
||||
An object id is used alone when a class requires arguments
|
||||
to it's __new__ method, which is signalled by the class having a
|
||||
__getnewargs__ attribute.
|
||||
|
||||
A number of legacyforms are defined:
|
||||
|
||||
|
||||
"""
|
||||
import logging
|
||||
|
||||
from persistent import Persistent
|
||||
from persistent.wref import WeakRefMarker, WeakRef
|
||||
from ZODB import broken
|
||||
from ZODB.POSException import InvalidObjectReference
|
||||
from ZODB._compat import PersistentPickler, PersistentUnpickler, BytesIO
|
||||
from ZODB._compat import _protocol, binary
|
||||
|
||||
|
||||
_oidtypes = bytes, type(None)
|
||||
|
||||
|
||||
# Might to update or redo coptimizations to reflect weakrefs:
|
||||
# from ZODB.coptimizations import new_persistent_id
|
||||
|
||||
def myhasattr(obj, name, _marker=object()):
|
||||
"""Make sure we don't mask exceptions like hasattr().
|
||||
|
||||
We don't want exceptions other than AttributeError to be masked,
|
||||
since that too often masks other programming errors.
|
||||
Three-argument getattr() doesn't mask those, so we use that to
|
||||
implement our own hasattr() replacement.
|
||||
"""
|
||||
return getattr(obj, name, _marker) is not _marker
|
||||
|
||||
|
||||
class ObjectWriter(object):
|
||||
"""Serializes objects for storage in the database.
|
||||
|
||||
The ObjectWriter creates object pickles in the ZODB format. It
|
||||
also detects new persistent objects reachable from the current
|
||||
object.
|
||||
"""
|
||||
|
||||
_jar = None
|
||||
|
||||
def __init__(self, obj=None):
|
||||
self._file = BytesIO()
|
||||
self._p = PersistentPickler(self.persistent_id, self._file, _protocol)
|
||||
self._stack = []
|
||||
if obj is not None:
|
||||
self._stack.append(obj)
|
||||
jar = obj._p_jar
|
||||
assert myhasattr(jar, "new_oid")
|
||||
self._jar = jar
|
||||
|
||||
def persistent_id(self, obj):
|
||||
"""Return the persistent id for obj.
|
||||
|
||||
>>> from ZODB.tests.util import P
|
||||
>>> class DummyJar(object):
|
||||
... xrefs = True
|
||||
... def new_oid(self):
|
||||
... return b'42'
|
||||
... def db(self):
|
||||
... return self
|
||||
... databases = {}
|
||||
|
||||
>>> jar = DummyJar()
|
||||
>>> class O(object):
|
||||
... _p_jar = jar
|
||||
>>> writer = ObjectWriter(O)
|
||||
|
||||
Normally, object references include the oid and a cached named
|
||||
reference to the class. Having the class information
|
||||
available allows fast creation of the ghost, avoiding
|
||||
requiring an additional database lookup.
|
||||
|
||||
>>> bob = P('bob')
|
||||
>>> oid, cls = writer.persistent_id(bob)
|
||||
>>> oid
|
||||
'42'
|
||||
>>> cls is P
|
||||
True
|
||||
|
||||
To work with Python 3, the oid in the persistent id is of the
|
||||
zodbpickle binary type:
|
||||
|
||||
>>> oid.__class__ is binary
|
||||
True
|
||||
|
||||
|
||||
If a persistent object does not already have an oid and jar,
|
||||
these will be assigned by persistent_id():
|
||||
|
||||
>>> bob._p_oid
|
||||
'42'
|
||||
>>> bob._p_jar is jar
|
||||
True
|
||||
|
||||
If the object already has a persistent id, the id is not changed:
|
||||
|
||||
>>> bob._p_oid = b'24'
|
||||
>>> oid, cls = writer.persistent_id(bob)
|
||||
>>> oid
|
||||
'24'
|
||||
>>> cls is P
|
||||
True
|
||||
|
||||
If the jar doesn't match that of the writer, an error is raised:
|
||||
|
||||
>>> bob._p_jar = DummyJar()
|
||||
>>> writer.persistent_id(bob)
|
||||
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
InvalidObjectReference:
|
||||
('Attempt to store an object from a foreign database connection',
|
||||
<ZODB.serialize.DummyJar ...>, P(bob))
|
||||
|
||||
Constructor arguments used by __new__(), as returned by
|
||||
__getnewargs__(), can affect memory allocation, but may also
|
||||
change over the life of the object. This makes it useless to
|
||||
cache even the object's class.
|
||||
|
||||
>>> class PNewArgs(P):
|
||||
... def __getnewargs__(self):
|
||||
... return ()
|
||||
|
||||
>>> sam = PNewArgs('sam')
|
||||
>>> writer.persistent_id(sam)
|
||||
'42'
|
||||
>>> sam._p_oid
|
||||
'42'
|
||||
>>> sam._p_jar is jar
|
||||
True
|
||||
|
||||
Check that simple objects don't get accused of persistence:
|
||||
|
||||
>>> writer.persistent_id(42)
|
||||
>>> writer.persistent_id(object())
|
||||
|
||||
Check that a classic class doesn't get identified improperly:
|
||||
|
||||
>>> class ClassicClara(object):
|
||||
... pass
|
||||
>>> clara = ClassicClara()
|
||||
|
||||
>>> writer.persistent_id(clara)
|
||||
"""
|
||||
|
||||
# Most objects are not persistent. The following cheap test
|
||||
# identifies most of them. For these, we return None,
|
||||
# signalling that the object should be pickled normally.
|
||||
if not isinstance(obj, (Persistent, type, WeakRef)):
|
||||
# Not persistent, pickle normally
|
||||
return None
|
||||
|
||||
# Any persistent object must have an oid:
|
||||
try:
|
||||
oid = obj._p_oid
|
||||
except AttributeError:
|
||||
# Not persistent, pickle normally
|
||||
return None
|
||||
|
||||
if not (oid is None or isinstance(oid, bytes)):
|
||||
# Deserves a closer look:
|
||||
|
||||
# Make sure it's not a descriptor
|
||||
if hasattr(oid, '__get__'):
|
||||
# The oid is a descriptor. That means obj is a non-persistent
|
||||
# class whose instances are persistent, so ...
|
||||
# Not persistent, pickle normally
|
||||
return None
|
||||
|
||||
if oid is WeakRefMarker:
|
||||
# we have a weakref, see weakref.py
|
||||
|
||||
oid = obj.oid
|
||||
if oid is None:
|
||||
target = obj() # get the referenced object
|
||||
oid = target._p_oid
|
||||
if oid is None:
|
||||
# Here we are causing the object to be saved in
|
||||
# the database. One could argue that we shouldn't
|
||||
# do this, because a weakref should not cause an object
|
||||
# to be added. We'll be optimistic, though, and
|
||||
# assume that the object will be added eventually.
|
||||
|
||||
oid = self._jar.new_oid()
|
||||
target._p_jar = self._jar
|
||||
target._p_oid = oid
|
||||
self._stack.append(target)
|
||||
obj.oid = oid
|
||||
obj.dm = target._p_jar
|
||||
obj.database_name = obj.dm.db().database_name
|
||||
|
||||
oid = binary(oid)
|
||||
if obj.dm is self._jar:
|
||||
return ['w', (oid, )]
|
||||
else:
|
||||
return ['w', (oid, obj.database_name)]
|
||||
|
||||
|
||||
# Since we have an oid, we have either a persistent instance
|
||||
# (an instance of Persistent), or a persistent class.
|
||||
|
||||
# NOTE! Persistent classes don't (and can't) subclass persistent.
|
||||
|
||||
database_name = None
|
||||
|
||||
if oid is None:
|
||||
oid = obj._p_oid = self._jar.new_oid()
|
||||
obj._p_jar = self._jar
|
||||
self._stack.append(obj)
|
||||
|
||||
elif obj._p_jar is not self._jar:
|
||||
if not self._jar.db().xrefs:
|
||||
raise InvalidObjectReference(
|
||||
"Database %r doesn't allow implicit cross-database "
|
||||
"references" % self._jar.db().database_name,
|
||||
self._jar, obj)
|
||||
|
||||
try:
|
||||
otherdb = obj._p_jar.db()
|
||||
database_name = otherdb.database_name
|
||||
except AttributeError:
|
||||
otherdb = self
|
||||
|
||||
if self._jar.db().databases.get(database_name) is not otherdb:
|
||||
raise InvalidObjectReference(
|
||||
"Attempt to store an object from a foreign "
|
||||
"database connection", self._jar, obj,
|
||||
)
|
||||
|
||||
if self._jar.get_connection(database_name) is not obj._p_jar:
|
||||
raise InvalidObjectReference(
|
||||
"Attempt to store a reference to an object from "
|
||||
"a separate connection to the same database or "
|
||||
"multidatabase", self._jar, obj,
|
||||
)
|
||||
|
||||
# OK, we have an object from another database.
|
||||
# Lets make sure the object ws not *just* loaded.
|
||||
|
||||
if obj._p_jar._implicitlyAdding(oid):
|
||||
raise InvalidObjectReference(
|
||||
"A new object is reachable from multiple databases. "
|
||||
"Won't try to guess which one was correct!",
|
||||
self._jar, obj,
|
||||
)
|
||||
|
||||
oid = binary(oid)
|
||||
klass = type(obj)
|
||||
if hasattr(klass, '__getnewargs__'):
|
||||
# We don't want to save newargs in object refs.
|
||||
# It's possible that __getnewargs__ is degenerate and
|
||||
# returns (), but we don't want to have to deghostify
|
||||
# the object to find out.
|
||||
|
||||
# Note that this has the odd effect that, if the class has
|
||||
# __getnewargs__ of its own, we'll lose the optimization
|
||||
# of caching the class info.
|
||||
|
||||
if database_name is not None:
|
||||
return ['n', (database_name, oid)]
|
||||
|
||||
return oid
|
||||
|
||||
# Note that we never get here for persistent classes.
|
||||
# We'll use direct refs for normal classes.
|
||||
|
||||
if database_name is not None:
|
||||
return ['m', (database_name, oid, klass)]
|
||||
|
||||
return oid, klass
|
||||
|
||||
def serialize(self, obj):
|
||||
# We don't use __class__ here, because obj could be a persistent proxy.
|
||||
# We don't want to be fooled by proxies.
|
||||
klass = type(obj)
|
||||
|
||||
# We want to serialize persistent classes by name if they have
|
||||
# a non-None non-empty module so as not to have a direct
|
||||
# ref. This is important when copying. We probably want to
|
||||
# revisit this in the future.
|
||||
newargs = getattr(obj, "__getnewargs__", None)
|
||||
if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
|
||||
and klass.__module__):
|
||||
# This is a persistent class with a non-empty module. This
|
||||
# uses pickle format #3 or #7.
|
||||
klass = klass.__module__, klass.__name__
|
||||
if newargs is None:
|
||||
meta = klass, None
|
||||
else:
|
||||
meta = klass, newargs()
|
||||
elif newargs is None:
|
||||
# Pickle format #1.
|
||||
meta = klass
|
||||
else:
|
||||
# Pickle format #2.
|
||||
meta = klass, newargs()
|
||||
|
||||
return self._dump(meta, obj.__getstate__())
|
||||
|
||||
def _dump(self, classmeta, state):
|
||||
# To reuse the existing BytesIO object, we must reset
|
||||
# the file position to 0 and truncate the file after the
|
||||
# new pickle is written.
|
||||
self._file.seek(0)
|
||||
self._p.clear_memo()
|
||||
self._p.dump(classmeta)
|
||||
self._p.dump(state)
|
||||
self._file.truncate()
|
||||
return self._file.getvalue()
|
||||
|
||||
def __iter__(self):
|
||||
return NewObjectIterator(self._stack)
|
||||
|
||||
class NewObjectIterator(object):
|
||||
|
||||
# The pickler is used as a forward iterator when the connection
|
||||
# is looking for new objects to pickle.
|
||||
|
||||
def __init__(self, stack):
|
||||
self._stack = stack
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self._stack:
|
||||
elt = self._stack.pop()
|
||||
return elt
|
||||
else:
|
||||
raise StopIteration
|
||||
|
||||
next = __next__
|
||||
|
||||
class ObjectReader(object):
|
||||
|
||||
def __init__(self, conn=None, cache=None, factory=None):
|
||||
self._conn = conn
|
||||
self._cache = cache
|
||||
self._factory = factory
|
||||
|
||||
def _get_class(self, module, name):
|
||||
return self._factory(self._conn, module, name)
|
||||
|
||||
def _get_unpickler(self, pickle):
|
||||
file = BytesIO(pickle)
|
||||
|
||||
factory = self._factory
|
||||
conn = self._conn
|
||||
|
||||
def find_global(modulename, name):
|
||||
return factory(conn, modulename, name)
|
||||
unpickler = PersistentUnpickler(find_global, self._persistent_load, file)
|
||||
|
||||
return unpickler
|
||||
|
||||
loaders = {}
|
||||
|
||||
def _persistent_load(self, reference):
|
||||
if isinstance(reference, tuple):
|
||||
return self.load_persistent(*reference)
|
||||
elif isinstance(reference, (bytes, str)):
|
||||
return self.load_oid(reference)
|
||||
else:
|
||||
try:
|
||||
reference_type, args = reference
|
||||
except ValueError:
|
||||
# weakref
|
||||
return self.loaders['w'](self, *reference)
|
||||
else:
|
||||
return self.loaders[reference_type](self, *args)
|
||||
|
||||
def load_persistent(self, oid, klass):
|
||||
# Quick instance reference. We know all we need to know
|
||||
# to create the instance w/o hitting the db, so go for it!
|
||||
|
||||
if not isinstance(oid, bytes):
|
||||
assert isinstance(oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
oid = oid.encode('ascii')
|
||||
|
||||
obj = self._cache.get(oid, None)
|
||||
if obj is not None:
|
||||
return obj
|
||||
|
||||
if isinstance(klass, tuple):
|
||||
klass = self._get_class(*klass)
|
||||
|
||||
if issubclass(klass, broken.Broken):
|
||||
# We got a broken class. We might need to make it
|
||||
# PersistentBroken
|
||||
if not issubclass(klass, broken.PersistentBroken):
|
||||
klass = broken.persistentBroken(klass)
|
||||
|
||||
try:
|
||||
obj = klass.__new__(klass)
|
||||
except TypeError:
|
||||
# Couldn't create the instance. Maybe there's more
|
||||
# current data in the object's actual record!
|
||||
return self._conn.get(oid)
|
||||
|
||||
# TODO: should be done by connection
|
||||
self._cache.new_ghost(oid, obj)
|
||||
return obj
|
||||
|
||||
def load_multi_persistent(self, database_name, oid, klass):
|
||||
conn = self._conn.get_connection(database_name)
|
||||
# TODO, make connection _cache attr public
|
||||
reader = ObjectReader(conn, conn._cache, self._factory)
|
||||
return reader.load_persistent(oid, klass)
|
||||
|
||||
loaders['m'] = load_multi_persistent
|
||||
|
||||
|
||||
def load_persistent_weakref(self, oid, database_name=None):
|
||||
if not isinstance(oid, bytes):
|
||||
assert isinstance(oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
oid = oid.encode('ascii')
|
||||
obj = WeakRef.__new__(WeakRef)
|
||||
obj.oid = oid
|
||||
if database_name is None:
|
||||
obj.dm = self._conn
|
||||
else:
|
||||
obj.database_name = database_name
|
||||
try:
|
||||
obj.dm = self._conn.get_connection(database_name)
|
||||
except KeyError:
|
||||
# XXX Not sure what to do here. It seems wrong to
|
||||
# fail since this is a weak reference. For now we'll
|
||||
# just pretend that the target object has gone.
|
||||
pass
|
||||
return obj
|
||||
|
||||
loaders['w'] = load_persistent_weakref
|
||||
|
||||
def load_oid(self, oid):
|
||||
if not isinstance(oid, bytes):
|
||||
assert isinstance(oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
oid = oid.encode('ascii')
|
||||
obj = self._cache.get(oid, None)
|
||||
if obj is not None:
|
||||
return obj
|
||||
return self._conn.get(oid)
|
||||
|
||||
def load_multi_oid(self, database_name, oid):
|
||||
conn = self._conn.get_connection(database_name)
|
||||
# TODO, make connection _cache attr public
|
||||
reader = ObjectReader(conn, conn._cache, self._factory)
|
||||
return reader.load_oid(oid)
|
||||
|
||||
loaders['n'] = load_multi_oid
|
||||
|
||||
def getClassName(self, pickle):
|
||||
unpickler = self._get_unpickler(pickle)
|
||||
klass = unpickler.load()
|
||||
if isinstance(klass, tuple):
|
||||
klass, args = klass
|
||||
if isinstance(klass, tuple):
|
||||
# old style reference
|
||||
return "%s.%s" % klass
|
||||
return "%s.%s" % (klass.__module__, klass.__name__)
|
||||
|
||||
def getGhost(self, pickle):
|
||||
unpickler = self._get_unpickler(pickle)
|
||||
klass = unpickler.load()
|
||||
if isinstance(klass, tuple):
|
||||
# Here we have a separate class and args.
|
||||
# This could be an old record, so the class module ne a named
|
||||
# refernce
|
||||
klass, args = klass
|
||||
if isinstance(klass, tuple):
|
||||
# Old module_name, class_name tuple
|
||||
klass = self._get_class(*klass)
|
||||
|
||||
if args is None:
|
||||
args = ()
|
||||
else:
|
||||
# Definitely new style direct class reference
|
||||
args = ()
|
||||
|
||||
if issubclass(klass, broken.Broken):
|
||||
# We got a broken class. We might need to make it
|
||||
# PersistentBroken
|
||||
if not issubclass(klass, broken.PersistentBroken):
|
||||
klass = broken.persistentBroken(klass)
|
||||
|
||||
return klass.__new__(klass, *args)
|
||||
|
||||
def getState(self, pickle):
|
||||
unpickler = self._get_unpickler(pickle)
|
||||
try:
|
||||
unpickler.load() # skip the class metadata
|
||||
return unpickler.load()
|
||||
except EOFError as msg:
|
||||
log = logging.getLogger("ZODB.serialize")
|
||||
log.exception("Unpickling error: %r", pickle)
|
||||
raise
|
||||
|
||||
def setGhostState(self, obj, pickle):
|
||||
state = self.getState(pickle)
|
||||
obj.__setstate__(state)
|
||||
|
||||
|
||||
def referencesf(p, oids=None):
|
||||
"""Return a list of object ids found in a pickle
|
||||
|
||||
A list may be passed in, in which case, information is
|
||||
appended to it.
|
||||
|
||||
Only ordinary internal references are included.
|
||||
Weak and multi-database references are not included.
|
||||
"""
|
||||
|
||||
refs = []
|
||||
u = PersistentUnpickler(None, refs.append, BytesIO(p))
|
||||
u.noload()
|
||||
u.noload()
|
||||
|
||||
# Now we have a list of referencs. Need to convert to list of
|
||||
# oids:
|
||||
|
||||
if oids is None:
|
||||
oids = []
|
||||
|
||||
for reference in refs:
|
||||
if isinstance(reference, tuple):
|
||||
oid = reference[0]
|
||||
elif isinstance(reference, (bytes, str)):
|
||||
oid = reference
|
||||
else:
|
||||
assert isinstance(reference, list)
|
||||
continue
|
||||
|
||||
if not isinstance(oid, bytes):
|
||||
assert isinstance(oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
oid = oid.encode('ascii')
|
||||
|
||||
oids.append(oid)
|
||||
|
||||
return oids
|
||||
|
||||
oid_klass_loaders = {
|
||||
'w': lambda oid, database_name=None: None,
|
||||
}
|
||||
|
||||
def get_refs(a_pickle):
|
||||
"""Return oid and class information for references in a pickle
|
||||
|
||||
The result of a list of oid and class information tuples.
|
||||
If the reference doesn't contain class information, then the
|
||||
klass information is None.
|
||||
"""
|
||||
|
||||
refs = []
|
||||
u = PersistentUnpickler(None, refs.append, BytesIO(a_pickle))
|
||||
u.noload()
|
||||
u.noload()
|
||||
|
||||
# Now we have a list of references. Need to convert to list of
|
||||
# oids and class info:
|
||||
|
||||
result = []
|
||||
|
||||
for reference in refs:
|
||||
if isinstance(reference, tuple):
|
||||
oid, klass = reference
|
||||
elif isinstance(reference, (bytes, str)):
|
||||
data, klass = reference, None
|
||||
else:
|
||||
assert isinstance(reference, list)
|
||||
continue
|
||||
|
||||
if not isinstance(oid, bytes):
|
||||
assert isinstance(oid, str)
|
||||
# this happens on Python 3 when all bytes in the oid are < 0x80
|
||||
oid = oid.encode('ascii')
|
||||
|
||||
result.append((oid, klass))
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,4 @@
|
||||
<schema>
|
||||
<import package="ZODB"/>
|
||||
<section type="ZODB.storage" name="*" attribute="storage"/>
|
||||
</schema>
|
||||
@@ -0,0 +1,51 @@
|
||||
=========================
|
||||
Subtransactions in ZODB 3
|
||||
=========================
|
||||
|
||||
ZODB 3 provides limited support for subtransactions. Subtransactions
|
||||
are nested to *one* level. There are top-level transactions and
|
||||
subtransactions. When a transaction is committed, a flag is passed
|
||||
indicating whether it is a subtransaction or a top-level transaction.
|
||||
Consider the following exampler commit calls:
|
||||
|
||||
- ``commit()``
|
||||
|
||||
A regular top-level transaction is committed.
|
||||
|
||||
- ``commit(1)``
|
||||
|
||||
A subtransaction is committed. There is now one subtransaction of
|
||||
the current top-level transaction.
|
||||
|
||||
- ``commit(1)``
|
||||
|
||||
A subtransaction is committed. There are now two subtransactions of
|
||||
the current top-level transaction.
|
||||
|
||||
- ``abort(1)``
|
||||
|
||||
A subtransaction is aborted. There are still two subtransactions of
|
||||
the current top-level transaction; work done since the last
|
||||
``commit(1)`` call is discarded.
|
||||
|
||||
- ``commit()``
|
||||
|
||||
We now commit a top-level transaction. The work done in the previous
|
||||
two subtransactions *plus* work done since the last ``abort(1)`` call
|
||||
is saved.
|
||||
|
||||
- ``commit(1)``
|
||||
|
||||
A subtransaction is committed. There is now one subtransaction of
|
||||
the current top-level transaction.
|
||||
|
||||
- ``commit(1)``
|
||||
|
||||
A subtransaction is committed. There are now two subtransactions of
|
||||
the current top-level transaction.
|
||||
|
||||
- ``abort()``
|
||||
|
||||
We now abort a top-level transaction. We discard the work done in
|
||||
the previous two subtransactions *plus* work done since the last
|
||||
``commit(1)`` call.
|
||||
@@ -0,0 +1,392 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Run the basic tests for a storage as described in the official storage API
|
||||
|
||||
The most complete and most out-of-date description of the interface is:
|
||||
http://www.zope.org/Documentation/Developer/Models/ZODB/ZODB_Architecture_Storage_Interface_Info.html
|
||||
|
||||
All storages should be able to pass these tests.
|
||||
"""
|
||||
from ZODB import POSException
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.tests.StorageTestBase import zodb_unpickle, zodb_pickle
|
||||
|
||||
import threading
|
||||
import time
|
||||
import zope.interface
|
||||
import zope.interface.verify
|
||||
|
||||
from .. import utils
|
||||
|
||||
ZERO = b'\0'*8
|
||||
|
||||
class BasicStorage(object):
|
||||
def checkBasics(self):
|
||||
self.assertEqual(self._storage.lastTransaction(), ZERO)
|
||||
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self.assertRaises(POSException.StorageTransactionError,
|
||||
self._storage.tpc_begin, t)
|
||||
# Aborting is easy
|
||||
self._storage.tpc_abort(t)
|
||||
# Test a few expected exceptions when we're doing operations giving a
|
||||
# different Transaction object than the one we've begun on.
|
||||
self._storage.tpc_begin(t)
|
||||
self.assertRaises(
|
||||
POSException.StorageTransactionError,
|
||||
self._storage.store,
|
||||
ZERO, ZERO, b'', '', TransactionMetaData())
|
||||
|
||||
self.assertRaises(
|
||||
POSException.StorageTransactionError,
|
||||
self._storage.store,
|
||||
ZERO, 1, b'2', '', TransactionMetaData())
|
||||
|
||||
self.assertRaises(
|
||||
POSException.StorageTransactionError,
|
||||
self._storage.tpc_vote, TransactionMetaData())
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
def checkSerialIsNoneForInitialRevision(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
txn = TransactionMetaData()
|
||||
self._storage.tpc_begin(txn)
|
||||
# Use None for serial. Don't use _dostore() here because that coerces
|
||||
# serial=None to serial=ZERO.
|
||||
self._storage.store(oid, None, zodb_pickle(MinPO(11)),
|
||||
'', txn)
|
||||
self._storage.tpc_vote(txn)
|
||||
newrevid = self._storage.tpc_finish(txn)
|
||||
data, revid = utils.load_current(self._storage, oid)
|
||||
value = zodb_unpickle(data)
|
||||
eq(value, MinPO(11))
|
||||
eq(revid, newrevid)
|
||||
|
||||
def checkStore(self):
|
||||
revid = ZERO
|
||||
newrevid = self._dostore(revid=None)
|
||||
# Finish the transaction.
|
||||
self.assertNotEqual(newrevid, revid)
|
||||
|
||||
def checkStoreAndLoad(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
self._dostore(oid=oid, data=MinPO(7))
|
||||
data, revid = utils.load_current(self._storage, oid)
|
||||
value = zodb_unpickle(data)
|
||||
eq(value, MinPO(7))
|
||||
# Now do a bunch of updates to an object
|
||||
for i in range(13, 22):
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(i))
|
||||
# Now get the latest revision of the object
|
||||
data, revid = utils.load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(21))
|
||||
|
||||
def checkConflicts(self):
|
||||
oid = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid, data=MinPO(11))
|
||||
self._dostore(oid, revid=revid1, data=MinPO(12))
|
||||
self.assertRaises(POSException.ConflictError,
|
||||
self._dostore,
|
||||
oid, revid=revid1, data=MinPO(13))
|
||||
|
||||
def checkWriteAfterAbort(self):
|
||||
oid = self._storage.new_oid()
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
|
||||
# Now abort this transaction
|
||||
self._storage.tpc_abort(t)
|
||||
# Now start all over again
|
||||
oid = self._storage.new_oid()
|
||||
self._dostore(oid=oid, data=MinPO(6))
|
||||
|
||||
def checkAbortAfterVote(self):
|
||||
oid1 = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid=oid1, data=MinPO(-2))
|
||||
oid = self._storage.new_oid()
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
|
||||
# Now abort this transaction
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_abort(t)
|
||||
# Now start all over again
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid=oid, data=MinPO(6))
|
||||
|
||||
for oid, revid in [(oid1, revid1), (oid, revid)]:
|
||||
data, _revid = utils.load_current(self._storage, oid)
|
||||
self.assertEqual(revid, _revid)
|
||||
|
||||
def checkStoreTwoObjects(self):
|
||||
noteq = self.assertNotEqual
|
||||
p31, p32, p51, p52 = map(MinPO, (31, 32, 51, 52))
|
||||
oid1 = self._storage.new_oid()
|
||||
oid2 = self._storage.new_oid()
|
||||
noteq(oid1, oid2)
|
||||
revid1 = self._dostore(oid1, data=p31)
|
||||
revid2 = self._dostore(oid2, data=p51)
|
||||
noteq(revid1, revid2)
|
||||
revid3 = self._dostore(oid1, revid=revid1, data=p32)
|
||||
revid4 = self._dostore(oid2, revid=revid2, data=p52)
|
||||
noteq(revid3, revid4)
|
||||
|
||||
def checkGetTid(self):
|
||||
if not hasattr(self._storage, 'getTid'):
|
||||
return
|
||||
eq = self.assertEqual
|
||||
p41, p42 = map(MinPO, (41, 42))
|
||||
oid = self._storage.new_oid()
|
||||
self.assertRaises(KeyError, self._storage.getTid, oid)
|
||||
# Now store a revision
|
||||
revid1 = self._dostore(oid, data=p41)
|
||||
eq(revid1, self._storage.getTid(oid))
|
||||
# And another one
|
||||
revid2 = self._dostore(oid, revid=revid1, data=p42)
|
||||
eq(revid2, self._storage.getTid(oid))
|
||||
|
||||
def checkLen(self):
|
||||
# len(storage) reports the number of objects.
|
||||
# check it is zero when empty
|
||||
self.assertEqual(len(self._storage),0)
|
||||
# check it is correct when the storage contains two object.
|
||||
# len may also be zero, for storages that do not keep track
|
||||
# of this number
|
||||
self._dostore(data=MinPO(22))
|
||||
self._dostore(data=MinPO(23))
|
||||
self.assertTrue(len(self._storage) in [0,2])
|
||||
|
||||
def checkGetSize(self):
|
||||
self._dostore(data=MinPO(25))
|
||||
size = self._storage.getSize()
|
||||
# The storage API doesn't make any claims about what size
|
||||
# means except that it ought to be printable.
|
||||
str(size)
|
||||
|
||||
def checkNote(self):
|
||||
oid = self._storage.new_oid()
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
t.note(u'this is a test')
|
||||
self._storage.store(oid, ZERO, zodb_pickle(MinPO(5)), '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
def checkInterfaces(self):
|
||||
for iface in zope.interface.providedBy(self._storage):
|
||||
zope.interface.verify.verifyObject(iface, self._storage)
|
||||
|
||||
def checkMultipleEmptyTransactions(self):
|
||||
# There was a bug in handling empty transactions in mapping
|
||||
# storage that caused the commit lock not to be released. :(
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t) # Hung here before
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
def _do_store_in_separate_thread(self, oid, revid, voted):
|
||||
# We'll run the competing trans in a separate thread:
|
||||
thread = threading.Thread(name='T2',
|
||||
target=self._dostore, args=(oid,), kwargs=dict(revid=revid))
|
||||
thread.setDaemon(True)
|
||||
thread.start()
|
||||
thread.join(.1)
|
||||
return thread
|
||||
|
||||
def check_checkCurrentSerialInTransaction(self):
|
||||
oid = b'\0\0\0\0\0\0\0\xf0'
|
||||
tid = self._dostore(oid)
|
||||
tid2 = self._dostore(oid, revid=tid)
|
||||
data = b'cpersistent\nPersistent\nq\x01.N.' # a simple persistent obj
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# stale read
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
try:
|
||||
self._storage.store(b'\0\0\0\0\0\0\0\xf1',
|
||||
b'\0\0\0\0\0\0\0\0', data, '', t)
|
||||
self._storage.checkCurrentSerialInTransaction(oid, tid, t)
|
||||
self._storage.tpc_vote(t)
|
||||
except POSException.ReadConflictError as v:
|
||||
self.assertEqual(v.oid, oid)
|
||||
self.assertEqual(v.serials, (tid2, tid))
|
||||
else:
|
||||
if 0: self.assertTrue(False, "No conflict error")
|
||||
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# non-stale read, no stress. :)
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(b'\0\0\0\0\0\0\0\xf2',
|
||||
b'\0\0\0\0\0\0\0\0', data, '', t)
|
||||
self._storage.checkCurrentSerialInTransaction(oid, tid2, t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# non-stale read, competition after vote. The competing
|
||||
# transaction must produce a tid > this transaction's tid
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(b'\0\0\0\0\0\0\0\xf3',
|
||||
b'\0\0\0\0\0\0\0\0', data, '', t)
|
||||
self._storage.checkCurrentSerialInTransaction(oid, tid2, t)
|
||||
self._storage.tpc_vote(t)
|
||||
|
||||
# We'll run the competing trans in a separate thread:
|
||||
thread = self._do_store_in_separate_thread(oid, tid2, True)
|
||||
self._storage.tpc_finish(t)
|
||||
thread.join(33)
|
||||
|
||||
tid3 = utils.load_current(self._storage, oid)[1]
|
||||
self.assertTrue(tid3 >
|
||||
utils.load_current(
|
||||
self._storage, b'\0\0\0\0\0\0\0\xf3')[1])
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
# non-stale competing trans after checkCurrentSerialInTransaction
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(b'\0\0\0\0\0\0\0\xf4',
|
||||
b'\0\0\0\0\0\0\0\0', data, '', t)
|
||||
self._storage.checkCurrentSerialInTransaction(oid, tid3, t)
|
||||
|
||||
thread = self._do_store_in_separate_thread(oid, tid3, False)
|
||||
|
||||
# There are 2 possibilities:
|
||||
# 1. The store happens before this transaction completes,
|
||||
# in which case, the vote below fails.
|
||||
# 2. The store happens after this trans, in which case, the
|
||||
# tid of the object is greater than this transaction's tid.
|
||||
try:
|
||||
self._storage.tpc_vote(t)
|
||||
except POSException.ReadConflictError:
|
||||
thread.join() # OK :)
|
||||
else:
|
||||
self._storage.tpc_finish(t)
|
||||
thread.join()
|
||||
tid4 = utils.load_current(self._storage, oid)[1]
|
||||
self.assertTrue(
|
||||
tid4 >
|
||||
utils.load_current(self._storage, b'\0\0\0\0\0\0\0\xf4')[1])
|
||||
|
||||
|
||||
def check_tid_ordering_w_commit(self):
|
||||
|
||||
# It's important that storages always give a consistent
|
||||
# ordering for revisions, tids. This is most likely to fail
|
||||
# around commit. Here we'll do some basic tests to check this.
|
||||
|
||||
# We'll use threads to arrange for ordering to go wrong and
|
||||
# verify that a storage gets it right.
|
||||
|
||||
# First, some initial data.
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(ZERO, ZERO, b'x', '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
tids = []
|
||||
self._storage.tpc_finish(t, lambda tid: tids.append(tid))
|
||||
|
||||
# OK, now we'll start a new transaction, take it to finish,
|
||||
# and then block finish while we do some other operations.
|
||||
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(ZERO, tids[0], b'y', '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
|
||||
to_join = []
|
||||
def run_in_thread(func):
|
||||
t = threading.Thread(target=func)
|
||||
t.setDaemon(True)
|
||||
t.start()
|
||||
to_join.append(t)
|
||||
|
||||
started = threading.Event()
|
||||
finish = threading.Event()
|
||||
@run_in_thread
|
||||
def commit():
|
||||
def callback(tid):
|
||||
started.set()
|
||||
tids.append(tid)
|
||||
finish.wait()
|
||||
|
||||
self._storage.tpc_finish(t, callback)
|
||||
|
||||
results = {}
|
||||
started.wait()
|
||||
attempts = []
|
||||
attempts_cond = utils.Condition()
|
||||
|
||||
def update_attempts():
|
||||
with attempts_cond:
|
||||
attempts.append(1)
|
||||
attempts_cond.notifyAll()
|
||||
|
||||
|
||||
@run_in_thread
|
||||
def lastTransaction():
|
||||
update_attempts()
|
||||
results['lastTransaction'] = self._storage.lastTransaction()
|
||||
|
||||
@run_in_thread
|
||||
def load():
|
||||
update_attempts()
|
||||
results['load'] = utils.load_current(self._storage, ZERO)[1]
|
||||
|
||||
expected_attempts = 2
|
||||
|
||||
if hasattr(self._storage, 'getTid'):
|
||||
expected_attempts += 1
|
||||
@run_in_thread
|
||||
def getTid():
|
||||
update_attempts()
|
||||
results['getTid'] = self._storage.getTid(ZERO)
|
||||
|
||||
if hasattr(self._storage, 'lastInvalidations'):
|
||||
expected_attempts += 1
|
||||
@run_in_thread
|
||||
def lastInvalidations():
|
||||
update_attempts()
|
||||
invals = self._storage.lastInvalidations(1)
|
||||
if invals:
|
||||
results['lastInvalidations'] = invals[0][0]
|
||||
|
||||
with attempts_cond:
|
||||
while len(attempts) < expected_attempts:
|
||||
attempts_cond.wait()
|
||||
|
||||
time.sleep(.01) # for good measure :)
|
||||
finish.set()
|
||||
|
||||
for t in to_join:
|
||||
t.join(1)
|
||||
|
||||
self.assertEqual(results.pop('load'), tids[1])
|
||||
self.assertEqual(results.pop('lastTransaction'), tids[1])
|
||||
for m, tid in results.items():
|
||||
self.assertEqual(tid, tids[1])
|
||||
@@ -0,0 +1,176 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Tests for application-level conflict resolution."""
|
||||
|
||||
from ZODB import DB
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.POSException import ConflictError, UndoError
|
||||
from persistent import Persistent
|
||||
from transaction import TransactionManager
|
||||
|
||||
from ZODB.tests.StorageTestBase import zodb_unpickle, zodb_pickle
|
||||
|
||||
class PCounter(Persistent):
|
||||
|
||||
_value = 0
|
||||
|
||||
def __repr__(self):
|
||||
return "<PCounter %d>" % self._value
|
||||
|
||||
def inc(self, n=1):
|
||||
self._value = self._value + n
|
||||
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
savedDiff = savedState['_value'] - oldState['_value']
|
||||
newDiff = newState['_value'] - oldState['_value']
|
||||
|
||||
oldState['_value'] = oldState['_value'] + savedDiff + newDiff
|
||||
|
||||
return oldState
|
||||
|
||||
# Insecurity: What if _p_resolveConflict _thinks_ it resolved the
|
||||
# conflict, but did something wrong?
|
||||
|
||||
class PCounter2(PCounter):
|
||||
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
raise ConflictError
|
||||
|
||||
class PCounter3(PCounter):
|
||||
def _p_resolveConflict(self, oldState, savedState, newState):
|
||||
raise AttributeError("no attribute (testing conflict resolution)")
|
||||
|
||||
class PCounter4(PCounter):
|
||||
def _p_resolveConflict(self, oldState, savedState):
|
||||
raise RuntimeError("Can't get here; not enough args")
|
||||
|
||||
class ConflictResolvingStorage(object):
|
||||
|
||||
def checkResolve(self, resolvable=True):
|
||||
db = DB(self._storage)
|
||||
|
||||
t1 = TransactionManager()
|
||||
c1 = db.open(t1)
|
||||
o1 = c1.root()['p'] = (PCounter if resolvable else PCounter2)()
|
||||
o1.inc()
|
||||
t1.commit()
|
||||
|
||||
t2 = TransactionManager()
|
||||
c2 = db.open(t2)
|
||||
o2 = c2.root()['p']
|
||||
o2.inc(2)
|
||||
t2.commit()
|
||||
|
||||
o1.inc(3)
|
||||
try:
|
||||
t1.commit()
|
||||
except ConflictError as err:
|
||||
self.assertIn(".PCounter2,", str(err))
|
||||
self.assertEqual(o1._value, 3)
|
||||
else:
|
||||
self.assertTrue(resolvable, "Expected ConflictError")
|
||||
self.assertEqual(o1._value, 6)
|
||||
|
||||
t2.begin()
|
||||
self.assertEqual(o2._value, o1._value)
|
||||
|
||||
db.close()
|
||||
|
||||
def checkUnresolvable(self):
|
||||
self.checkResolve(False)
|
||||
|
||||
def checkZClassesArentResolved(self):
|
||||
from ZODB.ConflictResolution import find_global, BadClassName
|
||||
dummy_class_tuple = ('*foobar', ())
|
||||
self.assertRaises(BadClassName, find_global, '*foobar', ())
|
||||
|
||||
def checkBuggyResolve1(self):
|
||||
obj = PCounter3()
|
||||
obj.inc()
|
||||
|
||||
oid = self._storage.new_oid()
|
||||
|
||||
revid1 = self._dostoreNP(oid, data=zodb_pickle(obj))
|
||||
|
||||
obj.inc()
|
||||
obj.inc()
|
||||
# The effect of committing two transactions with the same
|
||||
# pickle is to commit two different transactions relative to
|
||||
# revid1 that add two to _value.
|
||||
revid2 = self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
|
||||
self.assertRaises(ConflictError,
|
||||
self._dostoreNP,
|
||||
oid, revid=revid1, data=zodb_pickle(obj))
|
||||
|
||||
def checkBuggyResolve2(self):
|
||||
obj = PCounter4()
|
||||
obj.inc()
|
||||
|
||||
oid = self._storage.new_oid()
|
||||
|
||||
revid1 = self._dostoreNP(oid, data=zodb_pickle(obj))
|
||||
|
||||
obj.inc()
|
||||
obj.inc()
|
||||
# The effect of committing two transactions with the same
|
||||
# pickle is to commit two different transactions relative to
|
||||
# revid1 that add two to _value.
|
||||
revid2 = self._dostoreNP(oid, revid=revid1, data=zodb_pickle(obj))
|
||||
self.assertRaises(ConflictError,
|
||||
self._dostoreNP,
|
||||
oid, revid=revid1, data=zodb_pickle(obj))
|
||||
|
||||
class ConflictResolvingTransUndoStorage(object):
|
||||
|
||||
def checkUndoConflictResolution(self):
|
||||
# This test is based on checkNotUndoable in the
|
||||
# TransactionalUndoStorage test suite. Except here, conflict
|
||||
# resolution should allow us to undo the transaction anyway.
|
||||
|
||||
obj = PCounter()
|
||||
obj.inc()
|
||||
oid = self._storage.new_oid()
|
||||
revid_a = self._dostore(oid, data=obj)
|
||||
obj.inc()
|
||||
revid_b = self._dostore(oid, revid=revid_a, data=obj)
|
||||
obj.inc()
|
||||
revid_c = self._dostore(oid, revid=revid_b, data=obj)
|
||||
# Start the undo
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[1]['id']
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.undo(tid, t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
def checkUndoUnresolvable(self):
|
||||
# This test is based on checkNotUndoable in the
|
||||
# TransactionalUndoStorage test suite. Except here, conflict
|
||||
# resolution should allow us to undo the transaction anyway.
|
||||
|
||||
obj = PCounter2()
|
||||
obj.inc()
|
||||
oid = self._storage.new_oid()
|
||||
revid_a = self._dostore(oid, data=obj)
|
||||
obj.inc()
|
||||
revid_b = self._dostore(oid, revid=revid_a, data=obj)
|
||||
obj.inc()
|
||||
revid_c = self._dostore(oid, revid=revid_b, data=obj)
|
||||
# Start the undo
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[1]['id']
|
||||
t = TransactionMetaData()
|
||||
self.assertRaises(UndoError, self._begin_undos_vote, t, tid)
|
||||
self._storage.tpc_abort(t)
|
||||
@@ -0,0 +1,75 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Do some minimal tests of data corruption"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import stat
|
||||
|
||||
import ZODB.FileStorage
|
||||
|
||||
from ZODB.utils import load_current
|
||||
|
||||
from .StorageTestBase import StorageTestBase
|
||||
|
||||
class FileStorageCorruptTests(StorageTestBase):
|
||||
|
||||
def setUp(self):
|
||||
StorageTestBase.setUp(self)
|
||||
self._storage = ZODB.FileStorage.FileStorage('Data.fs', create=1)
|
||||
|
||||
def _do_stores(self):
|
||||
oids = []
|
||||
for i in range(5):
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid)
|
||||
oids.append((oid, revid))
|
||||
return oids
|
||||
|
||||
def _check_stores(self, oids):
|
||||
for oid, revid in oids:
|
||||
data, s_revid = load_current(self._storage, oid)
|
||||
self.assertEqual(s_revid, revid)
|
||||
|
||||
def checkTruncatedIndex(self):
|
||||
oids = self._do_stores()
|
||||
self._close()
|
||||
|
||||
# truncation the index file
|
||||
self.assertTrue(os.path.exists('Data.fs.index'))
|
||||
f = open('Data.fs.index', 'rb+')
|
||||
f.seek(0, 2)
|
||||
size = f.tell()
|
||||
f.seek(size // 2)
|
||||
f.truncate()
|
||||
f.close()
|
||||
|
||||
self._storage = ZODB.FileStorage.FileStorage('Data.fs')
|
||||
self._check_stores(oids)
|
||||
|
||||
def checkCorruptedIndex(self):
|
||||
oids = self._do_stores()
|
||||
self._close()
|
||||
|
||||
# truncation the index file
|
||||
self.assertTrue(os.path.exists('Data.fs.index'))
|
||||
size = os.stat('Data.fs.index')[stat.ST_SIZE]
|
||||
f = open('Data.fs.index', 'rb+')
|
||||
while f.tell() < size:
|
||||
f.seek(random.randrange(1, size // 10), 1)
|
||||
f.write(b'\000')
|
||||
f.close()
|
||||
|
||||
self._storage = ZODB.FileStorage.FileStorage('Data.fs')
|
||||
self._check_stores(oids)
|
||||
@@ -0,0 +1,55 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Run the history() related tests for a storage.
|
||||
|
||||
Any storage that supports the history() method should be able to pass
|
||||
all these tests.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from time import time, sleep
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
|
||||
class HistoryStorage(object):
|
||||
def checkSimpleHistory(self):
|
||||
self._checkHistory((11, 12, 13))
|
||||
|
||||
def _checkHistory(self, data):
|
||||
start = time()
|
||||
# Store a couple of revisions of the object
|
||||
oid = self._storage.new_oid()
|
||||
self.assertRaises(KeyError,self._storage.history,oid)
|
||||
revids = [None]
|
||||
for data in data:
|
||||
if sys.platform == 'win32':
|
||||
# time.time() has a precision of 1ms on Windows.
|
||||
sleep(0.002)
|
||||
revids.append(self._dostore(oid, revids[-1], MinPO(data)))
|
||||
revids.reverse()
|
||||
del revids[-1]
|
||||
# Now get various snapshots of the object's history
|
||||
for i in range(1, 1 + len(revids)):
|
||||
h = self._storage.history(oid, size=i)
|
||||
self.assertEqual([d['tid'] for d in h], revids[:i])
|
||||
# Check results are sorted by timestamp, in descending order.
|
||||
if sys.platform == 'win32':
|
||||
# Same as above. This is also required in case this method is
|
||||
# called several times for the same storage.
|
||||
sleep(0.002)
|
||||
a = time()
|
||||
for d in h:
|
||||
b = a
|
||||
a = d['time']
|
||||
self.assertLess(a, b)
|
||||
self.assertLess(start, a)
|
||||
@@ -0,0 +1,130 @@
|
||||
Storage Support for external GC
|
||||
===============================
|
||||
|
||||
A storage that provides IExternalGC supports external garbage
|
||||
collectors by providing a deleteObject method that transactionally
|
||||
deletes an object.
|
||||
|
||||
A create_storage function is provided that creates a storage.
|
||||
|
||||
>>> storage = create_storage()
|
||||
>>> import ZODB.blob, transaction
|
||||
>>> db = ZODB.DB(storage)
|
||||
>>> conn = db.open()
|
||||
>>> conn.root()[0] = conn.root().__class__()
|
||||
>>> conn.root()[1] = ZODB.blob.Blob(b'some data')
|
||||
>>> transaction.commit()
|
||||
>>> oid0 = conn.root()[0]._p_oid
|
||||
>>> oid1 = conn.root()[1]._p_oid
|
||||
>>> del conn.root()[0]
|
||||
>>> del conn.root()[1]
|
||||
>>> transaction.commit()
|
||||
|
||||
At this point, object 0 and 1 is garbage, but it's still in the storage:
|
||||
|
||||
>>> p0, s0 = storage.load(oid0, '')
|
||||
>>> p1, s1 = storage.load(oid1, '')
|
||||
|
||||
The storage is configured not to gc on pack, so even if we pack, these
|
||||
objects won't go away:
|
||||
|
||||
>>> len(storage)
|
||||
3
|
||||
>>> import time
|
||||
>>> db.pack(time.time()+1)
|
||||
>>> len(storage)
|
||||
3
|
||||
>>> p0, s0 = storage.load(oid0, '')
|
||||
>>> p1, s1 = storage.load(oid1, '')
|
||||
|
||||
Now we'll use the new deleteObject API to delete the objects. We can't
|
||||
go through the database to do this, so we'll have to manage the
|
||||
transaction ourselves.
|
||||
|
||||
>>> from ZODB.Connection import TransactionMetaData
|
||||
>>> txn = TransactionMetaData()
|
||||
>>> storage.tpc_begin(txn)
|
||||
>>> storage.deleteObject(oid0, s0, txn)
|
||||
>>> storage.deleteObject(oid1, s1, txn)
|
||||
>>> _ = storage.tpc_vote(txn)
|
||||
>>> tid = storage.tpc_finish(txn)
|
||||
>>> tid == storage.lastTransaction()
|
||||
True
|
||||
|
||||
Now if we try to load data for the objects, we get a POSKeyError:
|
||||
|
||||
|
||||
>>> storage.load(oid0, '') # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
>>> storage.load(oid1, '') # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
We can still get the data if we load before the time we deleted.
|
||||
|
||||
>>> storage.loadBefore(oid0, conn.root()._p_serial) == (p0, s0, tid)
|
||||
True
|
||||
>>> storage.loadBefore(oid1, conn.root()._p_serial) == (p1, s1, tid)
|
||||
True
|
||||
>>> with open(storage.loadBlob(oid1, s1)) as fp: fp.read()
|
||||
'some data'
|
||||
|
||||
If we pack, however, the old data will be removed and the data will be
|
||||
gone:
|
||||
|
||||
>>> db.pack(time.time()+1)
|
||||
>>> len(db.storage)
|
||||
1
|
||||
|
||||
>>> time.sleep(.1)
|
||||
|
||||
>>> storage.load(oid0, '') # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
>>> storage.load(oid1, '') # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
>>> storage.loadBefore(oid0, conn.root()._p_serial) # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
>>> storage.loadBefore(oid1, conn.root()._p_serial) # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
>>> storage.loadBlob(oid1, s1) # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: ...
|
||||
|
||||
A conflict error is raised if the serial we provide to deleteObject
|
||||
isn't current:
|
||||
|
||||
>>> conn.root()[0] = conn.root().__class__()
|
||||
>>> transaction.commit()
|
||||
>>> oid = conn.root()[0]._p_oid
|
||||
>>> bad_serial = conn.root()[0]._p_serial
|
||||
>>> conn.root()[0].x = 1
|
||||
>>> transaction.commit()
|
||||
|
||||
>>> txn = TransactionMetaData()
|
||||
>>> storage.tpc_begin(txn)
|
||||
>>> storage.deleteObject(oid, bad_serial, txn); storage.tpc_vote(txn)
|
||||
... # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ConflictError: database conflict error ...
|
||||
|
||||
>>> storage.tpc_abort(txn)
|
||||
|
||||
>>> storage.close()
|
||||
@@ -0,0 +1,247 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Run tests against the iterator() interface for storages.
|
||||
|
||||
Any storage that supports the iterator() method should be able to pass
|
||||
all these tests.
|
||||
|
||||
"""
|
||||
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.tests.StorageTestBase import zodb_pickle, zodb_unpickle
|
||||
from ZODB.utils import U64, p64, load_current
|
||||
|
||||
import ZODB.blob
|
||||
|
||||
try:
|
||||
from itertools import izip as zip
|
||||
except ImportError:
|
||||
# Py3: zip() already returns an iterable.
|
||||
pass
|
||||
|
||||
class IteratorCompare(object):
|
||||
|
||||
def iter_verify(self, txniter, revids, val0):
|
||||
eq = self.assertEqual
|
||||
oid = self._oid
|
||||
val = val0
|
||||
for reciter, revid in zip(txniter, revids + [None]):
|
||||
eq(reciter.tid, revid)
|
||||
for rec in reciter:
|
||||
eq(rec.oid, oid)
|
||||
eq(rec.tid, revid)
|
||||
eq(zodb_unpickle(rec.data), MinPO(val))
|
||||
val = val + 1
|
||||
eq(val, val0 + len(revids))
|
||||
|
||||
|
||||
class IteratorStorage(IteratorCompare):
|
||||
|
||||
def checkSimpleIteration(self):
|
||||
# Store a bunch of revisions of a single object
|
||||
self._oid = oid = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid, data=MinPO(11))
|
||||
revid2 = self._dostore(oid, revid=revid1, data=MinPO(12))
|
||||
revid3 = self._dostore(oid, revid=revid2, data=MinPO(13))
|
||||
# Now iterate over all the transactions and compare carefully
|
||||
txniter = self._storage.iterator()
|
||||
self.iter_verify(txniter, [revid1, revid2, revid3], 11)
|
||||
|
||||
def checkUndoZombie(self):
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=MinPO(94))
|
||||
# Get the undo information
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]['id']
|
||||
# Undo the creation of the object, rendering it a zombie
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
oids = self._storage.undo(tid, t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
# Now attempt to iterator over the storage
|
||||
iter = self._storage.iterator()
|
||||
for txn in iter:
|
||||
for rec in txn:
|
||||
pass
|
||||
|
||||
# The last transaction performed an undo of the transaction that
|
||||
# created object oid. (As Barry points out, the object is now in the
|
||||
# George Bailey state.) Assert that the final data record contains
|
||||
# None in the data attribute.
|
||||
self.assertEqual(rec.oid, oid)
|
||||
self.assertEqual(rec.data, None)
|
||||
|
||||
def checkTransactionExtensionFromIterator(self):
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=MinPO(1))
|
||||
iter = self._storage.iterator()
|
||||
count = 0
|
||||
for txn in iter:
|
||||
self.assertEqual(txn.extension, {})
|
||||
count +=1
|
||||
self.assertEqual(count, 1)
|
||||
|
||||
def checkIterationIntraTransaction(self):
|
||||
# TODO: Try this test with logging enabled. If you see something
|
||||
# like
|
||||
#
|
||||
# ZODB FS FS21 warn: FileStorageTests.fs truncated, possibly due to
|
||||
# damaged records at 4
|
||||
#
|
||||
# Then the code in FileIterator.next() hasn't yet been fixed.
|
||||
# Should automate that check.
|
||||
oid = self._storage.new_oid()
|
||||
t = TransactionMetaData()
|
||||
data = zodb_pickle(MinPO(0))
|
||||
try:
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid, '\0'*8, data, '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
# Don't do tpc_finish yet
|
||||
it = self._storage.iterator()
|
||||
for x in it:
|
||||
pass
|
||||
finally:
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
def checkLoad_was_checkLoadEx(self):
|
||||
oid = self._storage.new_oid()
|
||||
self._dostore(oid, data=42)
|
||||
data, tid = load_current(self._storage, oid)
|
||||
self.assertEqual(zodb_unpickle(data), MinPO(42))
|
||||
match = False
|
||||
for txn in self._storage.iterator():
|
||||
for rec in txn:
|
||||
if rec.oid == oid and rec.tid == tid:
|
||||
self.assertEqual(txn.tid, tid)
|
||||
match = True
|
||||
if not match:
|
||||
self.fail("Could not find transaction with matching id")
|
||||
|
||||
def checkIterateRepeatedly(self):
|
||||
self._dostore()
|
||||
transactions = self._storage.iterator()
|
||||
self.assertEqual(1, len(list(transactions)))
|
||||
# The iterator can only be consumed once:
|
||||
self.assertEqual(0, len(list(transactions)))
|
||||
|
||||
def checkIterateRecordsRepeatedly(self):
|
||||
self._dostore()
|
||||
it = self._storage.iterator()
|
||||
tinfo = next(it)
|
||||
self.assertEqual(1, len(list(tinfo)))
|
||||
self.assertEqual(1, len(list(tinfo)))
|
||||
if hasattr(it, 'close'):
|
||||
it.close()
|
||||
|
||||
def checkIterateWhileWriting(self):
|
||||
self._dostore()
|
||||
iterator = self._storage.iterator()
|
||||
# We have one transaction with 1 modified object.
|
||||
txn_1 = next(iterator)
|
||||
self.assertEqual(1, len(list(txn_1)))
|
||||
|
||||
# We store another transaction with 1 object, the already running
|
||||
# iterator does not pick this up.
|
||||
self._dostore()
|
||||
with self.assertRaises(StopIteration):
|
||||
next(iterator)
|
||||
|
||||
|
||||
class ExtendedIteratorStorage(IteratorCompare):
|
||||
|
||||
def checkExtendedIteration(self):
|
||||
# Store a bunch of revisions of a single object
|
||||
self._oid = oid = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid, data=MinPO(11))
|
||||
revid2 = self._dostore(oid, revid=revid1, data=MinPO(12))
|
||||
revid3 = self._dostore(oid, revid=revid2, data=MinPO(13))
|
||||
revid4 = self._dostore(oid, revid=revid3, data=MinPO(14))
|
||||
# Note that the end points are included
|
||||
# Iterate over all of the transactions with explicit start/stop
|
||||
txniter = self._storage.iterator(revid1, revid4)
|
||||
self.iter_verify(txniter, [revid1, revid2, revid3, revid4], 11)
|
||||
# Iterate over some of the transactions with explicit start
|
||||
txniter = self._storage.iterator(revid3)
|
||||
self.iter_verify(txniter, [revid3, revid4], 13)
|
||||
# Iterate over some of the transactions with explicit stop
|
||||
txniter = self._storage.iterator(None, revid2)
|
||||
self.iter_verify(txniter, [revid1, revid2], 11)
|
||||
# Iterate over some of the transactions with explicit start+stop
|
||||
txniter = self._storage.iterator(revid2, revid3)
|
||||
self.iter_verify(txniter, [revid2, revid3], 12)
|
||||
# Specify an upper bound somewhere in between values
|
||||
revid3a = p64((U64(revid3) + U64(revid4)) // 2)
|
||||
txniter = self._storage.iterator(revid2, revid3a)
|
||||
self.iter_verify(txniter, [revid2, revid3], 12)
|
||||
# Specify a lower bound somewhere in between values.
|
||||
# revid2 == revid1+1 is very likely on Windows. Adding 1 before
|
||||
# dividing ensures that "the midpoint" we compute is strictly larger
|
||||
# than revid1.
|
||||
revid1a = p64((U64(revid1) + 1 + U64(revid2)) // 2)
|
||||
assert revid1 < revid1a
|
||||
txniter = self._storage.iterator(revid1a, revid3a)
|
||||
self.iter_verify(txniter, [revid2, revid3], 12)
|
||||
# Specify an empty range
|
||||
txniter = self._storage.iterator(revid3, revid2)
|
||||
self.iter_verify(txniter, [], 13)
|
||||
# Specify a singleton range
|
||||
txniter = self._storage.iterator(revid3, revid3)
|
||||
self.iter_verify(txniter, [revid3], 13)
|
||||
|
||||
|
||||
class IteratorDeepCompare(object):
|
||||
|
||||
def compare(self, storage1, storage2):
|
||||
eq = self.assertEqual
|
||||
iter1 = storage1.iterator()
|
||||
iter2 = storage2.iterator()
|
||||
for txn1, txn2 in zip(iter1, iter2):
|
||||
eq(txn1.tid, txn2.tid)
|
||||
eq(txn1.status, txn2.status)
|
||||
eq(txn1.user, txn2.user)
|
||||
eq(txn1.description, txn2.description)
|
||||
eq(txn1.extension, txn2.extension)
|
||||
itxn1 = iter(txn1)
|
||||
itxn2 = iter(txn2)
|
||||
for rec1, rec2 in zip(itxn1, itxn2):
|
||||
eq(rec1.oid, rec2.oid)
|
||||
eq(rec1.tid, rec2.tid)
|
||||
eq(rec1.data, rec2.data)
|
||||
if ZODB.blob.is_blob_record(rec1.data):
|
||||
try:
|
||||
fn1 = storage1.loadBlob(rec1.oid, rec1.tid)
|
||||
except ZODB.POSException.POSKeyError:
|
||||
self.assertRaises(
|
||||
ZODB.POSException.POSKeyError,
|
||||
storage2.loadBlob, rec1.oid, rec1.tid)
|
||||
else:
|
||||
fn2 = storage2.loadBlob(rec1.oid, rec1.tid)
|
||||
self.assertTrue(fn1 != fn2)
|
||||
with open(fn1, 'rb') as fp1:
|
||||
with open(fn2, 'rb') as fp2:
|
||||
eq(fp1.read(), fp2.read())
|
||||
|
||||
# Make sure there are no more records left in rec1 and rec2,
|
||||
# meaning they were the same length.
|
||||
# Additionally, check that we're backwards compatible to the
|
||||
# IndexError we used to raise before.
|
||||
self.assertRaises(StopIteration, next, itxn1)
|
||||
self.assertRaises(StopIteration, next, itxn2)
|
||||
# Make sure ther are no more records left in txn1 and txn2, meaning
|
||||
# they were the same length
|
||||
self.assertRaises(StopIteration, next, iter1)
|
||||
self.assertRaises(StopIteration, next, iter2)
|
||||
@@ -0,0 +1,244 @@
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from persistent.mapping import PersistentMapping
|
||||
import six
|
||||
import transaction
|
||||
|
||||
import ZODB
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.tests.StorageTestBase import zodb_pickle, zodb_unpickle
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.POSException import ConflictError
|
||||
|
||||
from ZODB.utils import load_current
|
||||
|
||||
SHORT_DELAY = 0.01
|
||||
|
||||
class TestThread(threading.Thread):
|
||||
"""Base class for defining threads that run from unittest.
|
||||
|
||||
If the thread exits with an uncaught exception, catch it and
|
||||
re-raise it when the thread is joined. The re-raise will cause
|
||||
the test to fail.
|
||||
|
||||
The subclass should define a runtest() method instead of a run()
|
||||
method.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
threading.Thread.__init__(self)
|
||||
self._exc_info = None
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.runtest()
|
||||
except:
|
||||
self._exc_info = sys.exc_info()
|
||||
|
||||
def join(self, timeout=None):
|
||||
threading.Thread.join(self, timeout)
|
||||
if self._exc_info:
|
||||
raise six.reraise(
|
||||
self._exc_info[0], self._exc_info[1], self._exc_info[2])
|
||||
|
||||
class ZODBClientThread(TestThread):
|
||||
|
||||
__super_init = TestThread.__init__
|
||||
|
||||
def __init__(self, db, test, commits=10, delay=SHORT_DELAY):
|
||||
self.__super_init()
|
||||
self.setDaemon(1)
|
||||
self.db = db
|
||||
self.test = test
|
||||
self.commits = commits
|
||||
self.delay = delay
|
||||
|
||||
def runtest(self):
|
||||
conn = self.db.open()
|
||||
conn.sync()
|
||||
root = conn.root()
|
||||
d = self.get_thread_dict(root)
|
||||
if d is None:
|
||||
self.test.fail()
|
||||
else:
|
||||
for i in range(self.commits):
|
||||
self.commit(d, i)
|
||||
self.test.assertEqual(sorted(d.keys()), list(range(self.commits)))
|
||||
conn.close()
|
||||
|
||||
def commit(self, d, num):
|
||||
d[num] = time.time()
|
||||
time.sleep(self.delay)
|
||||
transaction.commit()
|
||||
time.sleep(self.delay)
|
||||
|
||||
# Return a new PersistentMapping, and store it on the root object under
|
||||
# the name (.getName()) of the current thread.
|
||||
def get_thread_dict(self, root):
|
||||
# This is vicious: multiple threads are slamming changes into the
|
||||
# root object, then trying to read the root object, simultaneously
|
||||
# and without any coordination. Conflict errors are rampant. It
|
||||
# used to go around at most 10 times, but that fairly often failed
|
||||
# to make progress in the 7-thread tests on some test boxes. Going
|
||||
# around (at most) 1000 times was enough so that a 100-thread test
|
||||
# reliably passed on Tim's hyperthreaded WinXP box (but at the
|
||||
# original 10 retries, the same test reliably failed with 15 threads).
|
||||
name = self.getName()
|
||||
MAXRETRIES = 1000
|
||||
|
||||
for i in range(MAXRETRIES):
|
||||
try:
|
||||
root[name] = PersistentMapping()
|
||||
transaction.commit()
|
||||
break
|
||||
except ConflictError:
|
||||
root._p_jar.sync()
|
||||
else:
|
||||
raise ConflictError("Exceeded %d attempts to store" % MAXRETRIES)
|
||||
|
||||
for j in range(MAXRETRIES):
|
||||
try:
|
||||
return root.get(name)
|
||||
except ConflictError:
|
||||
root._p_jar.sync()
|
||||
|
||||
raise ConflictError("Exceeded %d attempts to read" % MAXRETRIES)
|
||||
|
||||
class StorageClientThread(TestThread):
|
||||
|
||||
__super_init = TestThread.__init__
|
||||
|
||||
def __init__(self, storage, test, commits=10, delay=SHORT_DELAY):
|
||||
self.__super_init()
|
||||
self.storage = storage
|
||||
self.test = test
|
||||
self.commits = commits
|
||||
self.delay = delay
|
||||
self.oids = {}
|
||||
|
||||
def runtest(self):
|
||||
for i in range(self.commits):
|
||||
self.dostore(i)
|
||||
self.check()
|
||||
|
||||
def check(self):
|
||||
for oid, revid in self.oids.items():
|
||||
data, serial = load_current(self.storage, oid)
|
||||
self.test.assertEqual(serial, revid)
|
||||
obj = zodb_unpickle(data)
|
||||
self.test.assertEqual(obj.value[0], self.getName())
|
||||
|
||||
def pause(self):
|
||||
time.sleep(self.delay)
|
||||
|
||||
def oid(self):
|
||||
oid = self.storage.new_oid()
|
||||
self.oids[oid] = None
|
||||
return oid
|
||||
|
||||
def dostore(self, i):
|
||||
data = zodb_pickle(MinPO((self.getName(), i)))
|
||||
t = TransactionMetaData()
|
||||
oid = self.oid()
|
||||
self.pause()
|
||||
|
||||
self.storage.tpc_begin(t)
|
||||
self.pause()
|
||||
|
||||
# Always create a new object, signified by None for revid
|
||||
self.storage.store(oid, None, data, '', t)
|
||||
self.pause()
|
||||
|
||||
self.storage.tpc_vote(t)
|
||||
self.pause()
|
||||
|
||||
revid = self.storage.tpc_finish(t)
|
||||
self.pause()
|
||||
self.oids[oid] = revid
|
||||
|
||||
class ExtStorageClientThread(StorageClientThread):
|
||||
|
||||
def runtest(self):
|
||||
# pick some other storage ops to execute, depending in part
|
||||
# on the features provided by the storage.
|
||||
names = ["do_load"]
|
||||
|
||||
storage = self.storage
|
||||
|
||||
try:
|
||||
supportsUndo = storage.supportsUndo
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
if supportsUndo():
|
||||
names += ["do_loadSerial", "do_undoLog", "do_iterator"]
|
||||
|
||||
ops = [getattr(self, meth) for meth in names]
|
||||
assert ops, "Didn't find an storage ops in %s" % self.storage
|
||||
# do a store to guarantee there's at least one oid in self.oids
|
||||
self.dostore(0)
|
||||
|
||||
for i in range(self.commits - 1):
|
||||
meth = random.choice(ops)
|
||||
meth()
|
||||
self.dostore(i)
|
||||
self.check()
|
||||
|
||||
def pick_oid(self):
|
||||
return random.choice(tuple(self.oids))
|
||||
|
||||
def do_load(self):
|
||||
oid = self.pick_oid()
|
||||
load_current(self.storage, oid)
|
||||
|
||||
def do_loadSerial(self):
|
||||
oid = self.pick_oid()
|
||||
self.storage.loadSerial(oid, self.oids[oid])
|
||||
|
||||
def do_undoLog(self):
|
||||
self.storage.undoLog(0, -20)
|
||||
|
||||
def do_iterator(self):
|
||||
try:
|
||||
iter = self.storage.iterator()
|
||||
except AttributeError:
|
||||
# It's hard to detect that a ZEO ClientStorage
|
||||
# doesn't have this method, but does have all the others.
|
||||
return
|
||||
for obj in iter:
|
||||
pass
|
||||
|
||||
class MTStorage(object):
|
||||
"Test a storage with multiple client threads executing concurrently."
|
||||
|
||||
def _checkNThreads(self, n, constructor, *args):
|
||||
threads = [constructor(*args) for i in range(n)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(60)
|
||||
for t in threads:
|
||||
self.assertFalse(t.isAlive(),
|
||||
"thread failed to finish in 60 seconds")
|
||||
|
||||
def check2ZODBThreads(self):
|
||||
db = ZODB.DB(self._storage)
|
||||
self._checkNThreads(2, ZODBClientThread, db, self)
|
||||
db.close()
|
||||
|
||||
def check7ZODBThreads(self):
|
||||
db = ZODB.DB(self._storage)
|
||||
self._checkNThreads(7, ZODBClientThread, db, self)
|
||||
db.close()
|
||||
|
||||
def check2StorageThreads(self):
|
||||
self._checkNThreads(2, StorageClientThread, self._storage, self)
|
||||
|
||||
def check7StorageThreads(self):
|
||||
self._checkNThreads(7, StorageClientThread, self._storage, self)
|
||||
|
||||
def check4ExtStorageThread(self):
|
||||
self._checkNThreads(4, ExtStorageClientThread, self._storage, self)
|
||||
@@ -0,0 +1,124 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) Zope Corporation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
"""An extension of MappingStorage that depends on polling.
|
||||
|
||||
Each Connection has its own view of the database. Polling updates each
|
||||
connection's view.
|
||||
"""
|
||||
|
||||
import ZODB.utils
|
||||
import ZODB.POSException
|
||||
from ZODB.interfaces import IMVCCStorage
|
||||
from ZODB.MappingStorage import MappingStorage
|
||||
from zope.interface import implementer
|
||||
|
||||
|
||||
@implementer(IMVCCStorage)
|
||||
class MVCCMappingStorage(MappingStorage):
|
||||
|
||||
def __init__(self, name="MVCC Mapping Storage"):
|
||||
MappingStorage.__init__(self, name=name)
|
||||
# _polled_tid contains the transaction ID at the last poll.
|
||||
self._polled_tid = b''
|
||||
self._data_snapshot = None # {oid->(state, tid)}
|
||||
self._main_lock = self._lock
|
||||
|
||||
def new_instance(self):
|
||||
"""Returns a storage instance that is a view of the same data.
|
||||
"""
|
||||
inst = MVCCMappingStorage(name=self.__name__)
|
||||
# All instances share the same OID data, transaction log, commit lock,
|
||||
# and OID sequence.
|
||||
inst._data = self._data
|
||||
inst._transactions = self._transactions
|
||||
inst._commit_lock = self._commit_lock
|
||||
inst.new_oid = self.new_oid
|
||||
inst.pack = self.pack
|
||||
inst.loadBefore = self.loadBefore
|
||||
inst._ltid = self._ltid
|
||||
inst._main_lock = self._lock
|
||||
return inst
|
||||
|
||||
@ZODB.utils.locked(MappingStorage.opened)
|
||||
def sync(self, force=False):
|
||||
self._data_snapshot = None
|
||||
|
||||
def release(self):
|
||||
pass
|
||||
|
||||
@ZODB.utils.locked(MappingStorage.opened)
|
||||
def load(self, oid, version=''):
|
||||
assert not version, "Versions are not supported"
|
||||
if self._data_snapshot is None:
|
||||
self.poll_invalidations()
|
||||
info = self._data_snapshot.get(oid)
|
||||
if info:
|
||||
return info
|
||||
raise ZODB.POSException.POSKeyError(oid)
|
||||
|
||||
def poll_invalidations(self):
|
||||
"""Poll the storage for changes by other connections.
|
||||
"""
|
||||
# prevent changes to _transactions and _data during analysis
|
||||
with self._main_lock:
|
||||
if self._transactions:
|
||||
new_tid = self._transactions.maxKey()
|
||||
else:
|
||||
new_tid = ZODB.utils.z64
|
||||
|
||||
# Copy the current data into a snapshot. This is obviously
|
||||
# very inefficient for large storages, but it's good for
|
||||
# tests.
|
||||
self._data_snapshot = {}
|
||||
for oid, tid_data in self._data.items():
|
||||
if tid_data:
|
||||
tid = tid_data.maxKey()
|
||||
self._data_snapshot[oid] = tid_data[tid], tid
|
||||
|
||||
if self._polled_tid:
|
||||
if self._polled_tid not in self._transactions:
|
||||
# This connection is so old that we can no longer enumerate
|
||||
# all the changes.
|
||||
self._polled_tid = new_tid
|
||||
return None
|
||||
|
||||
changed_oids = set()
|
||||
for tid, txn in self._transactions.items(
|
||||
self._polled_tid, new_tid,
|
||||
excludemin=True, excludemax=False):
|
||||
if txn.status == 'p':
|
||||
# This transaction has been packed, so it is no longer
|
||||
# possible to enumerate all changed oids.
|
||||
self._polled_tid = new_tid
|
||||
return None
|
||||
if tid == self._ltid:
|
||||
# ignore the transaction committed by this connection
|
||||
continue
|
||||
changed_oids.update(txn.data.keys())
|
||||
|
||||
self._polled_tid = self._ltid = new_tid
|
||||
return list(changed_oids)
|
||||
|
||||
def tpc_finish(self, transaction, func = lambda tid: None):
|
||||
self._data_snapshot = None
|
||||
return MappingStorage.tpc_finish(self, transaction, func)
|
||||
|
||||
def tpc_abort(self, transaction):
|
||||
self._data_snapshot = None
|
||||
MappingStorage.tpc_abort(self, transaction)
|
||||
|
||||
def pack(self, t, referencesf, gc=True):
|
||||
# prevent all concurrent commits during packing
|
||||
with self._commit_lock:
|
||||
MappingStorage.pack(self, t, referencesf, gc)
|
||||
@@ -0,0 +1,50 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""A minimal persistent object to use for tests"""
|
||||
from persistent import Persistent
|
||||
|
||||
class MinPO(Persistent):
|
||||
def __init__(self, value=None):
|
||||
self.value = value
|
||||
|
||||
def __cmp__(self, aMinPO):
|
||||
return cmp(self.value, aMinPO.value)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.value)
|
||||
|
||||
# Py3: Python 3 does not support cmp() anymore. This is insane!!
|
||||
|
||||
def __eq__(self, aMinPO):
|
||||
return self.value == aMinPO.value
|
||||
|
||||
def __lt__(self, aMinPO):
|
||||
return self.value < aMinPO.value
|
||||
|
||||
# @functools.total_ordering is not available in 2.6 :-(
|
||||
|
||||
def __ne__(self, aMinPO):
|
||||
return self.value != aMinPO.value
|
||||
|
||||
def __gt__(self, aMinPO):
|
||||
return self.value > aMinPO.value
|
||||
|
||||
def __le__(self, aMinPO):
|
||||
return self.value <= aMinPO.value
|
||||
|
||||
def __ge__(self, aMinPO):
|
||||
return self.value >= aMinPO.value
|
||||
|
||||
def __repr__(self):
|
||||
return "MinPO(%s)" % self.value
|
||||
@@ -0,0 +1,798 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Run some tests relevant for storages that support pack()."""
|
||||
from __future__ import print_function
|
||||
|
||||
import doctest
|
||||
import time
|
||||
|
||||
from persistent import Persistent
|
||||
from persistent.mapping import PersistentMapping
|
||||
from ZODB import DB
|
||||
from ZODB.POSException import ConflictError, StorageError
|
||||
from ZODB.serialize import referencesf
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.tests.MTStorage import TestThread
|
||||
from ZODB.tests.StorageTestBase import snooze
|
||||
from ZODB._compat import (loads, PersistentPickler, Pickler, Unpickler,
|
||||
BytesIO, _protocol)
|
||||
import transaction
|
||||
import ZODB.interfaces
|
||||
import ZODB.tests.util
|
||||
from ZODB.tests.util import time_monotonically_increases
|
||||
import zope.testing.setupstack
|
||||
|
||||
from ZODB.utils import load_current
|
||||
|
||||
ZERO = b'\0'*8
|
||||
|
||||
|
||||
# This class is for the root object. It must not contain a getoid() method
|
||||
# (really, attribute). The persistent pickling machinery -- in the dumps()
|
||||
# function below -- will pickle Root objects as normal, but any attributes
|
||||
# which reference persistent Object instances will get pickled as persistent
|
||||
# ids, not as the object's state. This makes the referencesf stuff work,
|
||||
# because it pickle sniffs for persistent ids (so we have to get those
|
||||
# persistent ids into the root object's pickle).
|
||||
class Root(object):
|
||||
pass
|
||||
|
||||
|
||||
# This is the persistent Object class. Because it has a getoid() method, the
|
||||
# persistent pickling machinery -- in the dumps() function below -- will
|
||||
# pickle the oid string instead of the object's actual state. Yee haw, this
|
||||
# stuff is deep. ;)
|
||||
class Object(object):
|
||||
def __init__(self, oid):
|
||||
self._oid = oid
|
||||
|
||||
def getoid(self):
|
||||
return self._oid
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__dict__.clear()
|
||||
self.__dict__.update(state)
|
||||
if not isinstance(self._oid, bytes):
|
||||
# Python 3
|
||||
self._oid = self._oid.encode('ascii')
|
||||
|
||||
|
||||
class C(Persistent):
|
||||
pass
|
||||
|
||||
# Here's where all the magic occurs. Sadly, the pickle module is a bit
|
||||
# underdocumented, but here's what happens: by setting the persistent_id
|
||||
# attribute to getpersid() on the pickler, that function gets called for every
|
||||
# object being pickled. By returning None when the object has no getoid
|
||||
# attribute, it signals pickle to serialize the object as normal. That's how
|
||||
# the Root instance gets pickled correctly. But, if the object has a getoid
|
||||
# attribute, then by returning that method's value, we tell pickle to
|
||||
# serialize the persistent id of the object instead of the object's state.
|
||||
# That sets the pickle up for proper sniffing by the referencesf machinery.
|
||||
# Fun, huh?
|
||||
def dumps(obj):
|
||||
def getpersid(obj):
|
||||
if hasattr(obj, 'getoid'):
|
||||
return obj.getoid()
|
||||
return None
|
||||
s = BytesIO()
|
||||
p = PersistentPickler(getpersid, s, _protocol)
|
||||
p.dump(obj)
|
||||
p.dump(None)
|
||||
return s.getvalue()
|
||||
|
||||
def pdumps(obj):
|
||||
s = BytesIO()
|
||||
p = Pickler(s, _protocol)
|
||||
p.dump(obj)
|
||||
p.dump(None)
|
||||
return s.getvalue()
|
||||
|
||||
|
||||
class PackableStorageBase(object):
|
||||
# We keep a cache of object ids to instances so that the unpickler can
|
||||
# easily return any persistent object.
|
||||
|
||||
@property
|
||||
def _cache(self):
|
||||
try:
|
||||
return self.__cache
|
||||
except AttributeError:
|
||||
self.__cache = {}
|
||||
return self.__cache
|
||||
|
||||
def _newobj(self):
|
||||
# This is a convenience method to create a new persistent Object
|
||||
# instance. It asks the storage for a new object id, creates the
|
||||
# instance with the given oid, populates the cache and returns the
|
||||
# object.
|
||||
oid = self._storage.new_oid()
|
||||
obj = Object(oid)
|
||||
self._cache[obj.getoid()] = obj
|
||||
return obj
|
||||
|
||||
def _makeloader(self):
|
||||
# This is the other side of the persistent pickling magic. We need a
|
||||
# custom unpickler to mirror our custom pickler above. By setting the
|
||||
# persistent_load function of the unpickler to self._cache.get(),
|
||||
# whenever a persistent id is unpickled, it will actually return the
|
||||
# Object instance out of the cache. As far as returning a function
|
||||
# with an argument bound to an instance attribute method, we do it
|
||||
# this way because it makes the code in the tests more succinct.
|
||||
#
|
||||
# BUT! Be careful in your use of loads() vs. pickle.loads(). loads()
|
||||
# should only be used on the Root object's pickle since it's the only
|
||||
# special one. All the Object instances should use pickle.loads().
|
||||
def loads(str, persfunc=self._cache.get):
|
||||
fp = BytesIO(str)
|
||||
u = Unpickler(fp)
|
||||
u.persistent_load = persfunc
|
||||
return u.load()
|
||||
return loads
|
||||
|
||||
def _initroot(self):
|
||||
try:
|
||||
load_current(self._storage, ZERO)
|
||||
except KeyError:
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
file = BytesIO()
|
||||
p = Pickler(file, _protocol)
|
||||
p.dump((PersistentMapping, None))
|
||||
p.dump({'_container': {}})
|
||||
t = TransactionMetaData()
|
||||
t.description = u'initial database creation'
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(ZERO, None, file.getvalue(), '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
def _sanity_check(self):
|
||||
# Iterate over the storage to make sure it's sane.
|
||||
if not ZODB.interfaces.IStorageIteration.providedBy(self._storage):
|
||||
return
|
||||
it = self._storage.iterator()
|
||||
for txn in it:
|
||||
for data in txn:
|
||||
pass
|
||||
|
||||
|
||||
class PackableStorage(PackableStorageBase):
|
||||
|
||||
def checkPackEmptyStorage(self):
|
||||
self._storage.pack(time.time(), referencesf)
|
||||
|
||||
def checkPackTomorrow(self):
|
||||
self._initroot()
|
||||
self._storage.pack(time.time() + 10000, referencesf)
|
||||
|
||||
def checkPackYesterday(self):
|
||||
self._initroot()
|
||||
self._storage.pack(time.time() - 10000, referencesf)
|
||||
|
||||
def _PackWhileWriting(self, pack_now):
|
||||
# A storage should allow some reading and writing during
|
||||
# a pack. This test attempts to exercise locking code
|
||||
# in the storage to test that it is safe. It generates
|
||||
# a lot of revisions, so that pack takes a long time.
|
||||
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
|
||||
for i in range(10):
|
||||
root[i] = MinPO(i)
|
||||
transaction.commit()
|
||||
|
||||
snooze()
|
||||
packt = time.time()
|
||||
|
||||
choices = list(range(10))
|
||||
for dummy in choices:
|
||||
for i in choices:
|
||||
root[i].value = MinPO(i)
|
||||
transaction.commit()
|
||||
|
||||
# How many client threads should we run, and how long should we
|
||||
# wait for them to finish? Hard to say. Running 4 threads and
|
||||
# waiting 30 seconds too often left a thread still alive on Tim's
|
||||
# Win98SE box, during ZEO flavors of this test. Those tend to
|
||||
# run one thread at a time to completion, and take about 10 seconds
|
||||
# per thread. There doesn't appear to be a compelling reason to
|
||||
# run that many threads. Running 3 threads and waiting up to a
|
||||
# minute seems to work well in practice. The ZEO tests normally
|
||||
# finish faster than that, and the non-ZEO tests very much faster
|
||||
# than that.
|
||||
NUM_LOOP_TRIP = 50
|
||||
timer = ElapsedTimer(time.time())
|
||||
threads = [ClientThread(db, choices, NUM_LOOP_TRIP, timer, i)
|
||||
for i in range(3)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
if pack_now:
|
||||
db.pack(time.time())
|
||||
else:
|
||||
db.pack(packt)
|
||||
|
||||
for t in threads:
|
||||
t.join(60)
|
||||
liveness = [t.isAlive() for t in threads]
|
||||
if True in liveness:
|
||||
# They should have finished by now.
|
||||
print('Liveness:', liveness)
|
||||
# Combine the outcomes, and sort by start time.
|
||||
outcomes = []
|
||||
for t in threads:
|
||||
outcomes.extend(t.outcomes)
|
||||
# each outcome list has as many of these as a loop trip got thru:
|
||||
# thread_id
|
||||
# elapsed millis at loop top
|
||||
# elapsed millis at attempt to assign to self.root[index]
|
||||
# index into self.root getting replaced
|
||||
# elapsed millis when outcome known
|
||||
# 'OK' or 'Conflict'
|
||||
# True if we got beyond this line, False if it raised an
|
||||
# exception (one possible Conflict cause):
|
||||
# self.root[index].value = MinPO(j)
|
||||
def cmp_by_time(a, b):
|
||||
return cmp((a[1], a[0]), (b[1], b[0]))
|
||||
outcomes.sort(cmp_by_time)
|
||||
counts = [0] * 4
|
||||
for outcome in outcomes:
|
||||
n = len(outcome)
|
||||
assert n >= 2
|
||||
tid = outcome[0]
|
||||
print('tid:%d top:%5d' % (tid, outcome[1]), end=' ')
|
||||
if n > 2:
|
||||
print('commit:%5d' % outcome[2], end=' ')
|
||||
if n > 3:
|
||||
print('index:%2d' % outcome[3], end=' ')
|
||||
if n > 4:
|
||||
print('known:%5d' % outcome[4], end=' ')
|
||||
if n > 5:
|
||||
print('%8s' % outcome[5], end=' ')
|
||||
if n > 6:
|
||||
print('assigned:%5s' % outcome[6], end=' ')
|
||||
counts[tid] += 1
|
||||
if counts[tid] == NUM_LOOP_TRIP:
|
||||
print('thread %d done' % tid, end=' ')
|
||||
print()
|
||||
|
||||
self.fail('a thread is still alive')
|
||||
|
||||
self._sanity_check()
|
||||
|
||||
db.close()
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkPackWhileWriting(self):
|
||||
self._PackWhileWriting(pack_now=False)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkPackNowWhileWriting(self):
|
||||
self._PackWhileWriting(pack_now=True)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkPackLotsWhileWriting(self):
|
||||
# This is like the other pack-while-writing tests, except it packs
|
||||
# repeatedly until the client thread is done. At the time it was
|
||||
# introduced, it reliably provoked
|
||||
# CorruptedError: ... transaction with checkpoint flag set
|
||||
# in the ZEO flavor of the FileStorage tests.
|
||||
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
|
||||
choices = list(range(10))
|
||||
for i in choices:
|
||||
root[i] = MinPO(i)
|
||||
transaction.commit()
|
||||
|
||||
snooze()
|
||||
packt = time.time()
|
||||
|
||||
for dummy in choices:
|
||||
for i in choices:
|
||||
root[i].value = MinPO(i)
|
||||
transaction.commit()
|
||||
|
||||
NUM_LOOP_TRIP = 100
|
||||
timer = ElapsedTimer(time.time())
|
||||
thread = ClientThread(db, choices, NUM_LOOP_TRIP, timer, 0)
|
||||
thread.start()
|
||||
while thread.isAlive():
|
||||
db.pack(packt)
|
||||
snooze()
|
||||
packt = time.time()
|
||||
thread.join()
|
||||
|
||||
self._sanity_check()
|
||||
|
||||
db.close()
|
||||
|
||||
def checkPackWithMultiDatabaseReferences(self):
|
||||
databases = {}
|
||||
db = DB(self._storage, databases=databases, database_name='')
|
||||
otherdb = ZODB.tests.util.DB(databases=databases, database_name='o')
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
root[1] = C()
|
||||
transaction.commit()
|
||||
del root[1]
|
||||
transaction.commit()
|
||||
root[2] = conn.get_connection('o').root()
|
||||
transaction.commit()
|
||||
db.pack(time.time()+1)
|
||||
# some valid storages always return 0 for len()
|
||||
self.assertTrue(len(self._storage) in (0, 1))
|
||||
conn.close()
|
||||
otherdb.close()
|
||||
db.close()
|
||||
|
||||
def checkPackAllRevisions(self):
|
||||
self._initroot()
|
||||
eq = self.assertEqual
|
||||
raises = self.assertRaises
|
||||
# Create a `persistent' object
|
||||
obj = self._newobj()
|
||||
oid = obj.getoid()
|
||||
obj.value = 1
|
||||
# Commit three different revisions
|
||||
revid1 = self._dostoreNP(oid, data=pdumps(obj))
|
||||
obj.value = 2
|
||||
revid2 = self._dostoreNP(oid, revid=revid1, data=pdumps(obj))
|
||||
obj.value = 3
|
||||
revid3 = self._dostoreNP(oid, revid=revid2, data=pdumps(obj))
|
||||
# Now make sure all three revisions can be extracted
|
||||
data = self._storage.loadSerial(oid, revid1)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 1)
|
||||
data = self._storage.loadSerial(oid, revid2)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 2)
|
||||
data = self._storage.loadSerial(oid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 3)
|
||||
# Now pack all transactions; need to sleep a second to make
|
||||
# sure that the pack time is greater than the last commit time.
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# All revisions of the object should be gone, since there is no
|
||||
# reference from the root object to this object.
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid1)
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid2)
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid3)
|
||||
|
||||
def checkPackJustOldRevisions(self):
|
||||
eq = self.assertEqual
|
||||
raises = self.assertRaises
|
||||
loads = self._makeloader()
|
||||
# Create a root object. This can't be an instance of Object,
|
||||
# otherwise the pickling machinery will serialize it as a persistent
|
||||
# id and not as an object that contains references (persistent ids) to
|
||||
# other objects.
|
||||
root = Root()
|
||||
# Create a persistent object, with some initial state
|
||||
obj = self._newobj()
|
||||
oid = obj.getoid()
|
||||
# Link the root object to the persistent object, in order to keep the
|
||||
# persistent object alive. Store the root object.
|
||||
root.obj = obj
|
||||
root.value = 0
|
||||
revid0 = self._dostoreNP(ZERO, data=dumps(root))
|
||||
# Make sure the root can be retrieved
|
||||
data, revid = load_current(self._storage, ZERO)
|
||||
eq(revid, revid0)
|
||||
eq(loads(data).value, 0)
|
||||
# Commit three different revisions of the other object
|
||||
obj.value = 1
|
||||
revid1 = self._dostoreNP(oid, data=pdumps(obj))
|
||||
obj.value = 2
|
||||
revid2 = self._dostoreNP(oid, revid=revid1, data=pdumps(obj))
|
||||
obj.value = 3
|
||||
revid3 = self._dostoreNP(oid, revid=revid2, data=pdumps(obj))
|
||||
# Now make sure all three revisions can be extracted
|
||||
data = self._storage.loadSerial(oid, revid1)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 1)
|
||||
data = self._storage.loadSerial(oid, revid2)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 2)
|
||||
data = self._storage.loadSerial(oid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 3)
|
||||
# Now pack just revisions 1 and 2. The object's current revision
|
||||
# should stay alive because it's pointed to by the root.
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# Make sure the revisions are gone, but that object zero and revision
|
||||
# 3 are still there and correct
|
||||
data, revid = load_current(self._storage, ZERO)
|
||||
eq(revid, revid0)
|
||||
eq(loads(data).value, 0)
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid1)
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid2)
|
||||
data = self._storage.loadSerial(oid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 3)
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(revid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 3)
|
||||
|
||||
def checkPackOnlyOneObject(self):
|
||||
eq = self.assertEqual
|
||||
raises = self.assertRaises
|
||||
loads = self._makeloader()
|
||||
# Create a root object. This can't be an instance of Object,
|
||||
# otherwise the pickling machinery will serialize it as a persistent
|
||||
# id and not as an object that contains references (persistent ids) to
|
||||
# other objects.
|
||||
root = Root()
|
||||
# Create a persistent object, with some initial state
|
||||
obj1 = self._newobj()
|
||||
oid1 = obj1.getoid()
|
||||
# Create another persistent object, with some initial state.
|
||||
obj2 = self._newobj()
|
||||
oid2 = obj2.getoid()
|
||||
# Link the root object to the persistent objects, in order to keep
|
||||
# them alive. Store the root object.
|
||||
root.obj1 = obj1
|
||||
root.obj2 = obj2
|
||||
root.value = 0
|
||||
revid0 = self._dostoreNP(ZERO, data=dumps(root))
|
||||
# Make sure the root can be retrieved
|
||||
data, revid = load_current(self._storage, ZERO)
|
||||
eq(revid, revid0)
|
||||
eq(loads(data).value, 0)
|
||||
# Commit three different revisions of the first object
|
||||
obj1.value = 1
|
||||
revid1 = self._dostoreNP(oid1, data=pdumps(obj1))
|
||||
obj1.value = 2
|
||||
revid2 = self._dostoreNP(oid1, revid=revid1, data=pdumps(obj1))
|
||||
obj1.value = 3
|
||||
revid3 = self._dostoreNP(oid1, revid=revid2, data=pdumps(obj1))
|
||||
# Now make sure all three revisions can be extracted
|
||||
data = self._storage.loadSerial(oid1, revid1)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid1)
|
||||
eq(pobj.value, 1)
|
||||
data = self._storage.loadSerial(oid1, revid2)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid1)
|
||||
eq(pobj.value, 2)
|
||||
data = self._storage.loadSerial(oid1, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid1)
|
||||
eq(pobj.value, 3)
|
||||
# Now commit a revision of the second object
|
||||
obj2.value = 11
|
||||
revid4 = self._dostoreNP(oid2, data=pdumps(obj2))
|
||||
# And make sure the revision can be extracted
|
||||
data = self._storage.loadSerial(oid2, revid4)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid2)
|
||||
eq(pobj.value, 11)
|
||||
# Now pack just revisions 1 and 2 of object1. Object1's current
|
||||
# revision should stay alive because it's pointed to by the root, as
|
||||
# should Object2's current revision.
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# Make sure the revisions are gone, but that object zero, object2, and
|
||||
# revision 3 of object1 are still there and correct.
|
||||
data, revid = load_current(self._storage, ZERO)
|
||||
eq(revid, revid0)
|
||||
eq(loads(data).value, 0)
|
||||
raises(KeyError, self._storage.loadSerial, oid1, revid1)
|
||||
raises(KeyError, self._storage.loadSerial, oid1, revid2)
|
||||
data = self._storage.loadSerial(oid1, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid1)
|
||||
eq(pobj.value, 3)
|
||||
data, revid = load_current(self._storage, oid1)
|
||||
eq(revid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid1)
|
||||
eq(pobj.value, 3)
|
||||
data, revid = load_current(self._storage, oid2)
|
||||
eq(revid, revid4)
|
||||
eq(loads(data).value, 11)
|
||||
data = self._storage.loadSerial(oid2, revid4)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid2)
|
||||
eq(pobj.value, 11)
|
||||
|
||||
class PackableStorageWithOptionalGC(PackableStorage):
|
||||
|
||||
def checkPackAllRevisionsNoGC(self):
|
||||
self._initroot()
|
||||
eq = self.assertEqual
|
||||
raises = self.assertRaises
|
||||
# Create a `persistent' object
|
||||
obj = self._newobj()
|
||||
oid = obj.getoid()
|
||||
obj.value = 1
|
||||
# Commit three different revisions
|
||||
revid1 = self._dostoreNP(oid, data=pdumps(obj))
|
||||
obj.value = 2
|
||||
revid2 = self._dostoreNP(oid, revid=revid1, data=pdumps(obj))
|
||||
obj.value = 3
|
||||
revid3 = self._dostoreNP(oid, revid=revid2, data=pdumps(obj))
|
||||
# Now make sure all three revisions can be extracted
|
||||
data = self._storage.loadSerial(oid, revid1)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 1)
|
||||
data = self._storage.loadSerial(oid, revid2)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 2)
|
||||
data = self._storage.loadSerial(oid, revid3)
|
||||
pobj = loads(data)
|
||||
eq(pobj.getoid(), oid)
|
||||
eq(pobj.value, 3)
|
||||
# Now pack all transactions; need to sleep a second to make
|
||||
# sure that the pack time is greater than the last commit time.
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
self._storage.pack(packtime, referencesf, gc=False)
|
||||
# Only old revisions of the object should be gone. We don't gc
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid1)
|
||||
raises(KeyError, self._storage.loadSerial, oid, revid2)
|
||||
self._storage.loadSerial(oid, revid3)
|
||||
|
||||
|
||||
|
||||
class PackableUndoStorage(PackableStorageBase):
|
||||
|
||||
def checkPackUnlinkedFromRoot(self):
|
||||
eq = self.assertEqual
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
|
||||
txn = transaction.get()
|
||||
txn.note(u'root')
|
||||
txn.commit()
|
||||
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
|
||||
obj = C()
|
||||
obj.value = 7
|
||||
|
||||
root['obj'] = obj
|
||||
txn = transaction.get()
|
||||
txn.note(u'root -> o1')
|
||||
txn.commit()
|
||||
|
||||
del root['obj']
|
||||
txn = transaction.get()
|
||||
txn.note(u'root -x-> o1')
|
||||
txn.commit()
|
||||
|
||||
self._storage.pack(packtime, referencesf)
|
||||
|
||||
log = self._storage.undoLog()
|
||||
tid = log[0]['id']
|
||||
db.undo(tid)
|
||||
txn = transaction.get()
|
||||
txn.note(u'undo root -x-> o1')
|
||||
txn.commit()
|
||||
|
||||
conn.sync()
|
||||
|
||||
eq(root['obj'].value, 7)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkRedundantPack(self):
|
||||
# It is an error to perform a pack with a packtime earlier
|
||||
# than a previous packtime. The storage can't do a full
|
||||
# traversal as of the packtime, because the previous pack may
|
||||
# have removed revisions necessary for a full traversal.
|
||||
|
||||
# It should be simple to test that a storage error is raised,
|
||||
# but this test case goes to the trouble of constructing a
|
||||
# scenario that would lose data if the earlier packtime was
|
||||
# honored.
|
||||
|
||||
self._initroot()
|
||||
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
|
||||
root["d"] = d = PersistentMapping()
|
||||
transaction.commit()
|
||||
snooze()
|
||||
|
||||
obj = d["obj"] = C()
|
||||
obj.value = 1
|
||||
transaction.commit()
|
||||
snooze()
|
||||
packt1 = time.time()
|
||||
lost_oid = obj._p_oid
|
||||
|
||||
obj = d["anotherobj"] = C()
|
||||
obj.value = 2
|
||||
transaction.commit()
|
||||
snooze()
|
||||
packt2 = time.time()
|
||||
|
||||
db.pack(packt2)
|
||||
# BDBStorage allows the second pack, but doesn't lose data.
|
||||
try:
|
||||
db.pack(packt1)
|
||||
except StorageError:
|
||||
pass
|
||||
# This object would be removed by the second pack, even though
|
||||
# it is reachable.
|
||||
load_current(self._storage, lost_oid)
|
||||
|
||||
@time_monotonically_increases(0.1)
|
||||
def checkPackUndoLog(self):
|
||||
self._initroot()
|
||||
# Create a `persistent' object
|
||||
obj = self._newobj()
|
||||
oid = obj.getoid()
|
||||
obj.value = 1
|
||||
# Commit two different revisions
|
||||
revid1 = self._dostoreNP(oid, data=pdumps(obj))
|
||||
obj.value = 2
|
||||
snooze()
|
||||
packtime = time.time()
|
||||
snooze()
|
||||
self._dostoreNP(oid, revid=revid1, data=pdumps(obj))
|
||||
# Now pack the first transaction
|
||||
self.assertEqual(3, len(self._storage.undoLog()))
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# The undo log contains only the most resent transaction
|
||||
self.assertEqual(1, len(self._storage.undoLog()))
|
||||
|
||||
def dont_checkPackUndoLogUndoable(self):
|
||||
# A disabled test. I wanted to test that the content of the
|
||||
# undo log was consistent, but every storage appears to
|
||||
# include something slightly different. If the result of this
|
||||
# method is only used to fill a GUI then this difference
|
||||
# doesnt matter. Perhaps re-enable this test once we agree
|
||||
# what should be asserted.
|
||||
|
||||
self._initroot()
|
||||
# Create two `persistent' object
|
||||
obj1 = self._newobj()
|
||||
oid1 = obj1.getoid()
|
||||
obj1.value = 1
|
||||
obj2 = self._newobj()
|
||||
oid2 = obj2.getoid()
|
||||
obj2.value = 2
|
||||
|
||||
# Commit the first revision of each of them
|
||||
revid11 = self._dostoreNP(oid1, data=pdumps(obj1),
|
||||
description="1-1")
|
||||
revid22 = self._dostoreNP(oid2, data=pdumps(obj2),
|
||||
description="2-2")
|
||||
|
||||
# remember the time. everything above here will be packed away
|
||||
snooze()
|
||||
packtime = time.time()
|
||||
snooze()
|
||||
# Commit two revisions of the first object
|
||||
obj1.value = 3
|
||||
revid13 = self._dostoreNP(oid1, revid=revid11,
|
||||
data=pdumps(obj1), description="1-3")
|
||||
obj1.value = 4
|
||||
self._dostoreNP(oid1, revid=revid13,
|
||||
data=pdumps(obj1), description="1-4")
|
||||
# Commit one revision of the second object
|
||||
obj2.value = 5
|
||||
self._dostoreNP(oid2, revid=revid22,
|
||||
data=pdumps(obj2), description="2-5")
|
||||
# Now pack
|
||||
self.assertEqual(6,len(self._storage.undoLog()))
|
||||
print('\ninitial undoLog was')
|
||||
for r in self._storage.undoLog(): print(r)
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# The undo log contains only two undoable transaction.
|
||||
print('\nafter packing undoLog was')
|
||||
for r in self._storage.undoLog(): print(r)
|
||||
# what can we assert about that?
|
||||
|
||||
|
||||
# A number of these threads are kicked off by _PackWhileWriting(). Their
|
||||
# purpose is to abuse the database passed to the constructor with lots of
|
||||
# random write activity while the main thread is packing it.
|
||||
class ClientThread(TestThread):
|
||||
|
||||
def __init__(self, db, choices, loop_trip, timer, thread_id):
|
||||
TestThread.__init__(self)
|
||||
self.db = db
|
||||
self.choices = choices
|
||||
self.loop_trip = loop_trip
|
||||
self.millis = timer.elapsed_millis
|
||||
self.thread_id = thread_id
|
||||
# list of lists; each list has as many of these as a loop trip
|
||||
# got thru:
|
||||
# thread_id
|
||||
# elapsed millis at loop top
|
||||
# elapsed millis at attempt
|
||||
# index into self.root getting replaced
|
||||
# elapsed millis when outcome known
|
||||
# 'OK' or 'Conflict'
|
||||
# True if we got beyond this line, False if it raised an exception:
|
||||
# self.root[index].value = MinPO(j)
|
||||
self.outcomes = []
|
||||
|
||||
def runtest(self):
|
||||
from random import choice
|
||||
conn = self.db.open()
|
||||
|
||||
for j in range(self.loop_trip):
|
||||
assign_worked = False
|
||||
alist = [self.thread_id, self.millis()]
|
||||
self.outcomes.append(alist)
|
||||
try:
|
||||
index = choice(self.choices)
|
||||
alist.extend([self.millis(), index])
|
||||
conn.root()[index].value = MinPO(j)
|
||||
assign_worked = True
|
||||
transaction.commit()
|
||||
alist.append(self.millis())
|
||||
alist.append('OK')
|
||||
except ConflictError:
|
||||
alist.append(self.millis())
|
||||
alist.append('Conflict')
|
||||
transaction.abort()
|
||||
alist.append(assign_worked)
|
||||
|
||||
conn.close()
|
||||
|
||||
class ElapsedTimer(object):
|
||||
def __init__(self, start_time):
|
||||
self.start_time = start_time
|
||||
|
||||
def elapsed_millis(self):
|
||||
return int((time.time() - self.start_time) * 1000)
|
||||
|
||||
|
||||
def IExternalGC_suite(factory):
|
||||
"""Return a test suite for a generic .
|
||||
|
||||
Pass a factory taking a name and a blob directory name.
|
||||
"""
|
||||
|
||||
def setup(test):
|
||||
ZODB.tests.util.setUp(test)
|
||||
test.globs['create_storage'] = factory
|
||||
|
||||
return doctest.DocFileSuite(
|
||||
'IExternalGC.test',
|
||||
setUp=setup, tearDown=ZODB.tests.util.tearDown,
|
||||
checker=ZODB.tests.util.checker)
|
||||
@@ -0,0 +1,51 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Test that a storage's values persist across open and close."""
|
||||
|
||||
from ZODB.utils import load_current
|
||||
|
||||
class PersistentStorage(object):
|
||||
|
||||
def checkUpdatesPersist(self):
|
||||
oids = []
|
||||
|
||||
def new_oid_wrapper(l=oids, new_oid=self._storage.new_oid):
|
||||
oid = new_oid()
|
||||
l.append(oid)
|
||||
return oid
|
||||
|
||||
self._storage.new_oid = new_oid_wrapper
|
||||
|
||||
self._dostore()
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid)
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=1)
|
||||
revid = self._dostore(oid, revid, data=2)
|
||||
self._dostore(oid, revid, data=3)
|
||||
|
||||
# keep copies of all the objects
|
||||
objects = []
|
||||
for oid in oids:
|
||||
p, s = load_current(self._storage, oid)
|
||||
objects.append((oid, '', p, s))
|
||||
|
||||
self._storage.close()
|
||||
self.open()
|
||||
|
||||
# keep copies of all the objects
|
||||
for oid, ver, p, s in objects:
|
||||
_p, _s = load_current(self._storage, oid)
|
||||
self.assertEqual(p, _p)
|
||||
self.assertEqual(s, _s)
|
||||
@@ -0,0 +1,58 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.POSException import ReadOnlyError, Unsupported
|
||||
|
||||
from ZODB.utils import load_current
|
||||
|
||||
class ReadOnlyStorage(object):
|
||||
|
||||
def _create_data(self):
|
||||
# test a read-only storage that already has some data
|
||||
self.oids = {}
|
||||
for i in range(10):
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid)
|
||||
self.oids[oid] = revid
|
||||
|
||||
def _make_readonly(self):
|
||||
self._storage.close()
|
||||
self.open(read_only=True)
|
||||
self.assertTrue(self._storage.isReadOnly())
|
||||
|
||||
def checkReadMethods(self):
|
||||
self._create_data()
|
||||
self._make_readonly()
|
||||
# Note that this doesn't check _all_ read methods.
|
||||
for oid in self.oids.keys():
|
||||
data, revid = load_current(self._storage, oid)
|
||||
self.assertEqual(revid, self.oids[oid])
|
||||
# Storages without revisions may not have loadSerial().
|
||||
try:
|
||||
_data = self._storage.loadSerial(oid, revid)
|
||||
self.assertEqual(data, _data)
|
||||
except Unsupported:
|
||||
pass
|
||||
|
||||
def checkWriteMethods(self):
|
||||
self._make_readonly()
|
||||
self.assertRaises(ReadOnlyError, self._storage.new_oid)
|
||||
t = TransactionMetaData()
|
||||
self.assertRaises(ReadOnlyError, self._storage.tpc_begin, t)
|
||||
|
||||
self.assertRaises(ReadOnlyError, self._storage.store,
|
||||
b'\000' * 8, None, b'', '', t)
|
||||
|
||||
self.assertRaises(ReadOnlyError, self._storage.undo,
|
||||
b'\000' * 8, t)
|
||||
@@ -0,0 +1,200 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""More recovery and iterator tests."""
|
||||
|
||||
import transaction
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.tests.IteratorStorage import IteratorDeepCompare
|
||||
from ZODB.tests.StorageTestBase import MinPO, snooze
|
||||
from ZODB import DB
|
||||
from ZODB.serialize import referencesf
|
||||
|
||||
from ZODB.utils import load_current
|
||||
from ZODB.tests.util import time_monotonically_increases
|
||||
|
||||
import time
|
||||
|
||||
|
||||
class RecoveryStorage(IteratorDeepCompare):
|
||||
|
||||
# Requires a setUp() that creates a self._dst destination storage
|
||||
def checkSimpleRecovery(self):
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=11)
|
||||
revid = self._dostore(oid, revid=revid, data=12)
|
||||
revid = self._dostore(oid, revid=revid, data=13)
|
||||
self._dst.copyTransactionsFrom(self._storage)
|
||||
self.compare(self._storage, self._dst)
|
||||
|
||||
def checkRestoreAcrossPack(self):
|
||||
db = DB(self._storage)
|
||||
c = db.open()
|
||||
r = c.root()
|
||||
obj = r["obj1"] = MinPO(1)
|
||||
transaction.commit()
|
||||
obj = r["obj2"] = MinPO(1)
|
||||
transaction.commit()
|
||||
|
||||
self._dst.copyTransactionsFrom(self._storage)
|
||||
self._dst.pack(time.time(), referencesf)
|
||||
|
||||
self._undo(self._storage.undoInfo()[0]['id'])
|
||||
|
||||
# copy the final transaction manually. even though there
|
||||
# was a pack, the restore() ought to succeed.
|
||||
it = self._storage.iterator()
|
||||
# Get the last transaction and its record iterator. Record iterators
|
||||
# can't be accessed out-of-order, so we need to do this in a bit
|
||||
# complicated way:
|
||||
for final in it:
|
||||
records = list(final)
|
||||
|
||||
self._dst.tpc_begin(final, final.tid, final.status)
|
||||
for r in records:
|
||||
self._dst.restore(r.oid, r.tid, r.data, '', r.data_txn,
|
||||
final)
|
||||
self._dst.tpc_vote(final)
|
||||
self._dst.tpc_finish(final)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkPackWithGCOnDestinationAfterRestore(self):
|
||||
raises = self.assertRaises
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
root = conn.root()
|
||||
root.obj = obj1 = MinPO(1)
|
||||
txn = transaction.get()
|
||||
txn.note(u'root -> obj')
|
||||
txn.commit()
|
||||
root.obj.obj = obj2 = MinPO(2)
|
||||
txn = transaction.get()
|
||||
txn.note(u'root -> obj -> obj')
|
||||
txn.commit()
|
||||
del root.obj
|
||||
txn = transaction.get()
|
||||
txn.note(u'root -X->')
|
||||
txn.commit()
|
||||
# Now copy the transactions to the destination
|
||||
self._dst.copyTransactionsFrom(self._storage)
|
||||
# Now pack the destination.
|
||||
snooze()
|
||||
self._dst.pack(time.time(), referencesf)
|
||||
# And check to see that the root object exists, but not the other
|
||||
# objects.
|
||||
data, serial = load_current(self._dst, root._p_oid)
|
||||
raises(KeyError, load_current, self._dst, obj1._p_oid)
|
||||
raises(KeyError, load_current, self._dst, obj2._p_oid)
|
||||
|
||||
def checkRestoreWithMultipleObjectsInUndoRedo(self):
|
||||
from ZODB.FileStorage import FileStorage
|
||||
|
||||
# Undo creates backpointers in (at least) FileStorage. ZODB 3.2.1
|
||||
# FileStorage._data_find() had an off-by-8 error, neglecting to
|
||||
# account for the size of the backpointer when searching a
|
||||
# transaction with multiple data records. The results were
|
||||
# unpredictable. For example, it could raise a Python exception
|
||||
# due to passing a negative offset to file.seek(), or could
|
||||
# claim that a transaction didn't have data for an oid despite
|
||||
# that it actually did.
|
||||
#
|
||||
# The former failure mode was seen in real life, in a ZRS secondary
|
||||
# doing recovery. On my box today, the second failure mode is
|
||||
# what happens in this test (with an unpatched _data_find, of
|
||||
# course). Note that the error can only "bite" if more than one
|
||||
# data record is in a transaction, and the oid we're looking for
|
||||
# follows at least one data record with a backpointer.
|
||||
#
|
||||
# Unfortunately, _data_find() is a low-level implementation detail,
|
||||
# and this test does some horrid white-box abuse to test it.
|
||||
|
||||
is_filestorage = isinstance(self._storage, FileStorage)
|
||||
|
||||
db = DB(self._storage)
|
||||
c = db.open()
|
||||
r = c.root()
|
||||
|
||||
# Create some objects.
|
||||
r["obj1"] = MinPO(1)
|
||||
r["obj2"] = MinPO(1)
|
||||
transaction.commit()
|
||||
|
||||
# Add x attributes to them.
|
||||
r["obj1"].x = 'x1'
|
||||
r["obj2"].x = 'x2'
|
||||
transaction.commit()
|
||||
|
||||
r = db.open().root()
|
||||
self.assertEqual(r["obj1"].x, 'x1')
|
||||
self.assertEqual(r["obj2"].x, 'x2')
|
||||
|
||||
# Dirty tricks.
|
||||
if is_filestorage:
|
||||
obj1_oid = r["obj1"]._p_oid
|
||||
obj2_oid = r["obj2"]._p_oid
|
||||
# This will be the offset of the next transaction, which
|
||||
# will contain two backpointers.
|
||||
pos = self._storage.getSize()
|
||||
|
||||
# Undo the attribute creation.
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]['id']
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
oids = self._storage.undo(tid, t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
r = db.open().root()
|
||||
self.assertRaises(AttributeError, getattr, r["obj1"], 'x')
|
||||
self.assertRaises(AttributeError, getattr, r["obj2"], 'x')
|
||||
|
||||
if is_filestorage:
|
||||
# _data_find should find data records for both objects in that
|
||||
# transaction. Without the patch, the second assert failed
|
||||
# (it claimed it couldn't find a data record for obj2) on my
|
||||
# box, but other failure modes were possible.
|
||||
self.assertTrue(self._storage._data_find(pos, obj1_oid, '') > 0)
|
||||
self.assertTrue(self._storage._data_find(pos, obj2_oid, '') > 0)
|
||||
|
||||
# The offset of the next ("redo") transaction.
|
||||
pos = self._storage.getSize()
|
||||
|
||||
# Undo the undo (restore the attributes).
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]['id']
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
oids = self._storage.undo(tid, t)
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
|
||||
r = db.open().root()
|
||||
self.assertEqual(r["obj1"].x, 'x1')
|
||||
self.assertEqual(r["obj2"].x, 'x2')
|
||||
|
||||
if is_filestorage:
|
||||
# Again _data_find should find both objects in this txn, and
|
||||
# again the second assert failed on my box.
|
||||
self.assertTrue(self._storage._data_find(pos, obj1_oid, '') > 0)
|
||||
self.assertTrue(self._storage._data_find(pos, obj2_oid, '') > 0)
|
||||
|
||||
# Indirectly provoke .restore(). .restore in turn indirectly
|
||||
# provokes _data_find too, but not usefully for the purposes of
|
||||
# the specific bug this test aims at: copyTransactionsFrom() uses
|
||||
# storage iterators that chase backpointers themselves, and
|
||||
# return the data they point at instead. The result is that
|
||||
# _data_find didn't actually see anything dangerous in this
|
||||
# part of the test.
|
||||
self._dst.copyTransactionsFrom(self._storage)
|
||||
self.compare(self._storage, self._dst)
|
||||
@@ -0,0 +1,175 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Check loadSerial() on storages that support historical revisions."""
|
||||
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.tests.StorageTestBase import zodb_unpickle, zodb_pickle, snooze
|
||||
from ZODB.utils import p64, u64, load_current
|
||||
from ZODB.tests.util import time_monotonically_increases
|
||||
|
||||
ZERO = '\0'*8
|
||||
|
||||
class RevisionStorage(object):
|
||||
|
||||
def checkLoadSerial(self):
|
||||
oid = self._storage.new_oid()
|
||||
revid = ZERO
|
||||
revisions = {}
|
||||
for i in range(31, 38):
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(i))
|
||||
revisions[revid] = MinPO(i)
|
||||
# Now make sure all the revisions have the correct value
|
||||
for revid, value in revisions.items():
|
||||
data = self._storage.loadSerial(oid, revid)
|
||||
self.assertEqual(zodb_unpickle(data), value)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkLoadBefore(self):
|
||||
# Store 10 revisions of one object and then make sure that we
|
||||
# can get all the non-current revisions back.
|
||||
oid = self._storage.new_oid()
|
||||
revs = []
|
||||
revid = None
|
||||
for i in range(10):
|
||||
# We need to ensure that successive timestamps are at least
|
||||
# two apart, so that a timestamp exists that's unambiguously
|
||||
# between successive timestamps. Each call to snooze()
|
||||
# guarantees that the next timestamp will be at least one
|
||||
# larger (and probably much more than that) than the previous
|
||||
# one.
|
||||
snooze()
|
||||
snooze()
|
||||
revid = self._dostore(oid, revid, data=MinPO(i))
|
||||
revs.append(load_current(self._storage, oid))
|
||||
|
||||
prev = u64(revs[0][1])
|
||||
for i in range(1, 10):
|
||||
tid = revs[i][1]
|
||||
cur = u64(tid)
|
||||
middle = prev + (cur - prev) // 2
|
||||
assert prev < middle < cur # else the snooze() trick failed
|
||||
prev = cur
|
||||
t = self._storage.loadBefore(oid, p64(middle))
|
||||
self.assertTrue(t is not None)
|
||||
data, start, end = t
|
||||
self.assertEqual(revs[i-1][0], data)
|
||||
self.assertEqual(tid, end)
|
||||
|
||||
def checkLoadBeforeEdges(self):
|
||||
# Check the edges cases for a non-current load.
|
||||
oid = self._storage.new_oid()
|
||||
|
||||
self.assertRaises(KeyError, self._storage.loadBefore,
|
||||
oid, p64(0))
|
||||
|
||||
revid1 = self._dostore(oid, data=MinPO(1))
|
||||
|
||||
self.assertEqual(self._storage.loadBefore(oid, p64(0)), None)
|
||||
self.assertEqual(self._storage.loadBefore(oid, revid1), None)
|
||||
|
||||
cur = p64(u64(revid1) + 1)
|
||||
data, start, end = self._storage.loadBefore(oid, cur)
|
||||
self.assertEqual(zodb_unpickle(data), MinPO(1))
|
||||
self.assertEqual(start, revid1)
|
||||
self.assertEqual(end, None)
|
||||
|
||||
revid2 = self._dostore(oid, revid=revid1, data=MinPO(2))
|
||||
data, start, end = self._storage.loadBefore(oid, cur)
|
||||
self.assertEqual(zodb_unpickle(data), MinPO(1))
|
||||
self.assertEqual(start, revid1)
|
||||
self.assertEqual(end, revid2)
|
||||
|
||||
@time_monotonically_increases
|
||||
def checkLoadBeforeOld(self):
|
||||
# Look for a very old revision. With the BaseStorage implementation
|
||||
# this should require multple history() calls.
|
||||
oid = self._storage.new_oid()
|
||||
revs = []
|
||||
revid = None
|
||||
for i in range(50):
|
||||
revid = self._dostore(oid, revid, data=MinPO(i))
|
||||
revs.append(revid)
|
||||
|
||||
data, start, end = self._storage.loadBefore(oid, revs[12])
|
||||
self.assertEqual(zodb_unpickle(data), MinPO(11))
|
||||
self.assertEqual(start, revs[11])
|
||||
self.assertEqual(end, revs[12])
|
||||
|
||||
|
||||
# Unsure: Is it okay to assume everyone testing against RevisionStorage
|
||||
# implements undo?
|
||||
|
||||
def checkLoadBeforeUndo(self):
|
||||
# Do several transactions then undo them.
|
||||
oid = self._storage.new_oid()
|
||||
revid = None
|
||||
for i in range(5):
|
||||
revid = self._dostore(oid, revid, data=MinPO(i))
|
||||
revs = []
|
||||
for i in range(4):
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]["id"]
|
||||
# Always undo the most recent txn, so the value will
|
||||
# alternate between 3 and 4.
|
||||
self._undo(tid, note="undo %d" % i)
|
||||
revs.append(load_current(self._storage, oid))
|
||||
|
||||
prev_tid = None
|
||||
for i, (data, tid) in enumerate(revs):
|
||||
t = self._storage.loadBefore(oid, p64(u64(tid) + 1))
|
||||
self.assertEqual(data, t[0])
|
||||
self.assertEqual(tid, t[1])
|
||||
if prev_tid:
|
||||
self.assertTrue(prev_tid < t[1])
|
||||
prev_tid = t[1]
|
||||
if i < 3:
|
||||
self.assertEqual(revs[i+1][1], t[2])
|
||||
else:
|
||||
self.assertEqual(None, t[2])
|
||||
|
||||
def checkLoadBeforeConsecutiveTids(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
def helper(tid, revid, x):
|
||||
data = zodb_pickle(MinPO(x))
|
||||
t = TransactionMetaData()
|
||||
try:
|
||||
self._storage.tpc_begin(t, p64(tid))
|
||||
self._storage.store(oid, revid, data, '', t)
|
||||
# Finish the transaction
|
||||
self._storage.tpc_vote(t)
|
||||
newrevid = self._storage.tpc_finish(t)
|
||||
except:
|
||||
self._storage.tpc_abort(t)
|
||||
raise
|
||||
return newrevid
|
||||
revid1 = helper(1, None, 1)
|
||||
revid2 = helper(2, revid1, 2)
|
||||
revid3 = helper(3, revid2, 3)
|
||||
data, start_tid, end_tid = self._storage.loadBefore(oid, p64(2))
|
||||
eq(zodb_unpickle(data), MinPO(1))
|
||||
eq(u64(start_tid), 1)
|
||||
eq(u64(end_tid), 2)
|
||||
|
||||
def checkLoadBeforeCreation(self):
|
||||
eq = self.assertEqual
|
||||
oid1 = self._storage.new_oid()
|
||||
oid2 = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid1)
|
||||
revid2 = self._dostore(oid2)
|
||||
results = self._storage.loadBefore(oid2, revid2)
|
||||
eq(results, None)
|
||||
|
||||
# TODO: There are other edge cases to handle, including pack.
|
||||
@@ -0,0 +1,183 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Provide a mixin base class for storage tests.
|
||||
|
||||
The StorageTestBase class provides basic setUp() and tearDown()
|
||||
semantics (which you can override), and it also provides a helper
|
||||
method _dostore() which performs a complete store transaction for a
|
||||
single object revision.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
import time
|
||||
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.utils import u64, z64
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol
|
||||
import ZODB.tests.util
|
||||
|
||||
|
||||
ZERO = b'\0'*8
|
||||
|
||||
def snooze():
|
||||
# In Windows, it's possible that two successive time.time() calls return
|
||||
# the same value. Tim guarantees that time never runs backwards. You
|
||||
# usually want to call this before you pack a storage, or must make other
|
||||
# guarantees about increasing timestamps.
|
||||
now = time.time()
|
||||
while now == time.time():
|
||||
time.sleep(0.1)
|
||||
|
||||
def _persistent_id(obj):
|
||||
oid = getattr(obj, "_p_oid", None)
|
||||
if getattr(oid, "__get__", None) is not None:
|
||||
return None
|
||||
else:
|
||||
return oid
|
||||
|
||||
def zodb_pickle(obj):
|
||||
"""Create a pickle in the format expected by ZODB."""
|
||||
f = BytesIO()
|
||||
p = PersistentPickler(_persistent_id, f, _protocol)
|
||||
klass = obj.__class__
|
||||
assert not hasattr(obj, '__getinitargs__'), "not ready for constructors"
|
||||
args = None
|
||||
|
||||
mod = getattr(klass, '__module__', None)
|
||||
if mod is not None:
|
||||
klass = mod, klass.__name__
|
||||
|
||||
state = obj.__getstate__()
|
||||
|
||||
p.dump((klass, args))
|
||||
p.dump(state)
|
||||
return f.getvalue()
|
||||
|
||||
def persistent_load(pid):
|
||||
# helper for zodb_unpickle
|
||||
return "ref to %s.%s oid=%s" % (pid[1][0], pid[1][1], u64(pid[0]))
|
||||
|
||||
def zodb_unpickle(data):
|
||||
"""Unpickle an object stored using the format expected by ZODB."""
|
||||
f = BytesIO(data)
|
||||
u = Unpickler(f)
|
||||
u.persistent_load = persistent_load
|
||||
klass_info = u.load()
|
||||
if isinstance(klass_info, tuple):
|
||||
if isinstance(klass_info[0], type):
|
||||
# Unclear: what is the second part of klass_info?
|
||||
klass, xxx = klass_info
|
||||
assert not xxx
|
||||
else:
|
||||
if isinstance(klass_info[0], tuple):
|
||||
modname, klassname = klass_info[0]
|
||||
else:
|
||||
modname, klassname = klass_info
|
||||
if modname == "__main__":
|
||||
ns = globals()
|
||||
else:
|
||||
mod = import_helper(modname)
|
||||
ns = mod.__dict__
|
||||
try:
|
||||
klass = ns[klassname]
|
||||
except KeyError:
|
||||
print("can't find %s in %r" % (klassname, ns), file=sys.stderr)
|
||||
inst = klass()
|
||||
else:
|
||||
raise ValueError("expected class info: %s" % repr(klass_info))
|
||||
state = u.load()
|
||||
inst.__setstate__(state)
|
||||
return inst
|
||||
|
||||
def import_helper(name):
|
||||
__import__(name)
|
||||
return sys.modules[name]
|
||||
|
||||
|
||||
class StorageTestBase(ZODB.tests.util.TestCase):
|
||||
|
||||
# It would be simpler if concrete tests didn't need to extend
|
||||
# setUp() and tearDown().
|
||||
|
||||
_storage = None
|
||||
|
||||
def _close(self):
|
||||
# You should override this if closing your storage requires additional
|
||||
# shutdown operations.
|
||||
if self._storage is not None:
|
||||
self._storage.close()
|
||||
|
||||
def tearDown(self):
|
||||
self._close()
|
||||
ZODB.tests.util.TestCase.tearDown(self)
|
||||
|
||||
def _dostore(self, oid=None, revid=None, data=None,
|
||||
already_pickled=0, user=None, description=None):
|
||||
"""Do a complete storage transaction. The defaults are:
|
||||
|
||||
- oid=None, ask the storage for a new oid
|
||||
- revid=None, use a revid of ZERO
|
||||
- data=None, pickle up some arbitrary data (the integer 7)
|
||||
|
||||
Returns the object's new revision id.
|
||||
"""
|
||||
if oid is None:
|
||||
oid = self._storage.new_oid()
|
||||
if revid is None:
|
||||
revid = ZERO
|
||||
if data is None:
|
||||
data = MinPO(7)
|
||||
if type(data) == int:
|
||||
data = MinPO(data)
|
||||
if not already_pickled:
|
||||
data = zodb_pickle(data)
|
||||
# Begin the transaction
|
||||
t = TransactionMetaData()
|
||||
if user is not None:
|
||||
t.user = user
|
||||
if description is not None:
|
||||
t.description = description
|
||||
try:
|
||||
self._storage.tpc_begin(t)
|
||||
# Store an object
|
||||
r1 = self._storage.store(oid, revid, data, '', t)
|
||||
# Finish the transaction
|
||||
r2 = self._storage.tpc_vote(t)
|
||||
revid = self._storage.tpc_finish(t)
|
||||
except:
|
||||
self._storage.tpc_abort(t)
|
||||
raise
|
||||
return revid
|
||||
|
||||
def _dostoreNP(self, oid=None, revid=None, data=None,
|
||||
user=None, description=None):
|
||||
return self._dostore(oid, revid, data, 1, user, description)
|
||||
|
||||
# The following methods depend on optional storage features.
|
||||
|
||||
def _undo(self, tid, expected_oids=None, note=None):
|
||||
# Undo a tid that affects a single object (oid).
|
||||
# This is very specialized.
|
||||
t = TransactionMetaData()
|
||||
t.note(note or u"undo")
|
||||
self._storage.tpc_begin(t)
|
||||
undo_result = self._storage.undo(tid, t)
|
||||
vote_result = self._storage.tpc_vote(t)
|
||||
if expected_oids is not None:
|
||||
oids = set(undo_result[1]) if undo_result else set()
|
||||
if vote_result:
|
||||
oids.update(vote_result)
|
||||
self.assertEqual(oids, set(expected_oids))
|
||||
return self._storage.tpc_finish(t)
|
||||
@@ -0,0 +1,118 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Test the storage's implemenetation of the storage synchronization spec.
|
||||
|
||||
The Synchronization spec
|
||||
http://www.zope.org/Documentation/Developer/Models/ZODB/
|
||||
ZODB_Architecture_Storage_Interface_State_Synchronization_Diag.html
|
||||
|
||||
It specifies two states committing and non-committing. A storage
|
||||
starts in the non-committing state. tpc_begin() transfers to the
|
||||
committting state; tpc_abort() and tpc_finish() transfer back to
|
||||
non-committing.
|
||||
|
||||
Several other methods are only allowed in one state or another. Many
|
||||
methods allowed only in the committing state require that they apply
|
||||
to the currently committing transaction.
|
||||
|
||||
The spec is silent on a variety of methods that don't appear to modify
|
||||
the state, e.g. load(), undoLog(), pack(). It's unclear whether there
|
||||
is a separate set of synchronization rules that apply to these methods
|
||||
or if the synchronization is implementation dependent, i.e. only what
|
||||
is need to guarantee a corrected implementation.
|
||||
|
||||
The synchronization spec is also silent on whether there is any
|
||||
contract implied with the caller. If the storage can assume that a
|
||||
single client is single-threaded and that it will not call, e.g., store()
|
||||
until after it calls tpc_begin(), the implementation can be
|
||||
substantially simplified.
|
||||
|
||||
New and/or unspecified methods:
|
||||
|
||||
tpc_vote(): handled like tpc_abort
|
||||
undo(): how's that handled?
|
||||
|
||||
Methods that have nothing to do with committing/non-committing:
|
||||
load(), loadSerial(), getName(), getSize(), __len__(), history(),
|
||||
undoLog(), pack().
|
||||
|
||||
Specific questions:
|
||||
|
||||
The spec & docs say that undo() takes three arguments, the second
|
||||
being a transaction. If the specified arg isn't the current
|
||||
transaction, the undo() should raise StorageTransactionError. This
|
||||
isn't implemented anywhere. It looks like undo can be called at
|
||||
anytime.
|
||||
|
||||
FileStorage does not allow undo() during a pack. How should this be
|
||||
tested? Is it a general restriction?
|
||||
|
||||
|
||||
|
||||
"""
|
||||
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.POSException import StorageTransactionError
|
||||
|
||||
OID = "\000" * 8
|
||||
SERIALNO = "\000" * 8
|
||||
TID = "\000" * 8
|
||||
|
||||
class SynchronizedStorage(object):
|
||||
|
||||
def verifyNotCommitting(self, callable, *args):
|
||||
self.assertRaises(StorageTransactionError, callable, *args)
|
||||
|
||||
def verifyWrongTrans(self, callable, *args):
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self.assertRaises(StorageTransactionError, callable, *args)
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
def checkStoreNotCommitting(self):
|
||||
self.verifyNotCommitting(self._storage.store,
|
||||
OID, SERIALNO, b"", "", TransactionMetaData())
|
||||
|
||||
def checkStoreWrongTrans(self):
|
||||
self.verifyWrongTrans(self._storage.store,
|
||||
OID, SERIALNO, b"", "", TransactionMetaData())
|
||||
|
||||
def checkAbortNotCommitting(self):
|
||||
self._storage.tpc_abort(TransactionMetaData())
|
||||
|
||||
def checkAbortWrongTrans(self):
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.tpc_abort(TransactionMetaData())
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
def checkFinishNotCommitting(self):
|
||||
t = TransactionMetaData()
|
||||
self.assertRaises(StorageTransactionError,
|
||||
self._storage.tpc_finish, t)
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
def checkFinishWrongTrans(self):
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self.assertRaises(StorageTransactionError,
|
||||
self._storage.tpc_finish, TransactionMetaData())
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
def checkBeginCommitting(self):
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.tpc_abort(t)
|
||||
|
||||
# TODO: how to check undo?
|
||||
@@ -0,0 +1,773 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
"""Check undo().
|
||||
|
||||
Any storage that supports undo() must pass these tests.
|
||||
"""
|
||||
import time
|
||||
|
||||
from six import PY3
|
||||
|
||||
from persistent import Persistent
|
||||
import transaction
|
||||
from transaction import Transaction
|
||||
|
||||
from ZODB import POSException
|
||||
from ZODB.Connection import TransactionMetaData
|
||||
from ZODB.serialize import referencesf
|
||||
from ZODB.utils import p64, load_current
|
||||
from ZODB import DB
|
||||
|
||||
from ZODB.tests.MinPO import MinPO
|
||||
from ZODB.tests.StorageTestBase import zodb_pickle, zodb_unpickle
|
||||
|
||||
ZERO = '\0'*8
|
||||
|
||||
class C(Persistent):
|
||||
pass
|
||||
|
||||
def snooze():
|
||||
# In Windows, it's possible that two successive time.time() calls return
|
||||
# the same value. Tim guarantees that time never runs backwards. You
|
||||
# usually want to call this before you pack a storage, or must make other
|
||||
# guarantees about increasing timestamps.
|
||||
now = time.time()
|
||||
while now == time.time():
|
||||
time.sleep(0.1)
|
||||
|
||||
def listeq(L1, L2):
|
||||
"""Return True if L1.sort() == L2.sort()
|
||||
|
||||
Also support iterators.
|
||||
"""
|
||||
return sorted(L1) == sorted(L2)
|
||||
|
||||
class TransactionalUndoStorage(object):
|
||||
|
||||
def _multi_obj_transaction(self, objs):
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
for oid, rev, data in objs:
|
||||
self._storage.store(oid, rev, data, '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
return self._storage.tpc_finish(t)
|
||||
|
||||
def _iterate(self):
|
||||
"""Iterate over the storage in its final state."""
|
||||
# This is testing that the iterator() code works correctly.
|
||||
# The hasattr() guards against ZEO, which doesn't support iterator.
|
||||
if not hasattr(self._storage, "iterator"):
|
||||
return
|
||||
iter = self._storage.iterator()
|
||||
for txn in iter:
|
||||
for rec in txn:
|
||||
pass
|
||||
|
||||
def _begin_undos_vote(self, t, *tids):
|
||||
self._storage.tpc_begin(t)
|
||||
oids = set()
|
||||
for tid in tids:
|
||||
undo_result = self._storage.undo(tid, t)
|
||||
if undo_result:
|
||||
oids.update(undo_result[1])
|
||||
oids.update(self._storage.tpc_vote(t) or ())
|
||||
return oids
|
||||
|
||||
def undo(self, tid, note=None):
|
||||
t = TransactionMetaData()
|
||||
if note is not None:
|
||||
t.note(note)
|
||||
oids = self._begin_undos_vote(t, tid)
|
||||
self._storage.tpc_finish(t)
|
||||
return oids
|
||||
|
||||
def checkSimpleTransactionalUndo(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=MinPO(23))
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(24))
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(25))
|
||||
|
||||
info = self._storage.undoInfo()
|
||||
# Now start an undo transaction
|
||||
self._undo(info[0]["id"], [oid], note="undo1")
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(24))
|
||||
|
||||
# Do another one
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[2]["id"], [oid], note="undo2")
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(23))
|
||||
|
||||
# Try to undo the first record
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[4]["id"], [oid], note="undo3")
|
||||
# This should fail since we've undone the object's creation
|
||||
self.assertRaises(KeyError, load_current, self._storage, oid)
|
||||
|
||||
# And now let's try to redo the object's creation
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]["id"], [oid])
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(23))
|
||||
self._iterate()
|
||||
|
||||
def checkCreationUndoneGetTid(self):
|
||||
# create an object
|
||||
oid = self._storage.new_oid()
|
||||
self._dostore(oid, data=MinPO(23))
|
||||
# undo its creation
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]['id']
|
||||
self.undo(tid, 'undo1')
|
||||
# Check that calling getTid on an uncreated object raises a KeyError
|
||||
# The current version of FileStorage fails this test
|
||||
self.assertRaises(KeyError, self._storage.getTid, oid)
|
||||
|
||||
def checkUndoCreationBranch1(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=MinPO(11))
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(12))
|
||||
# Undo the last transaction
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]['id'], [oid])
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(11))
|
||||
|
||||
# Now from here, we can either redo the last undo, or undo the object
|
||||
# creation. Let's undo the object creation.
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[2]['id'], [oid])
|
||||
self.assertRaises(KeyError, load_current, self._storage, oid)
|
||||
|
||||
# Loading current data via loadBefore should raise a POSKeyError too:
|
||||
self.assertRaises(KeyError, self._storage.loadBefore, oid,
|
||||
b'\x7f\xff\xff\xff\xff\xff\xff\xff')
|
||||
self._iterate()
|
||||
|
||||
def checkUndoCreationBranch2(self):
|
||||
eq = self.assertEqual
|
||||
oid = self._storage.new_oid()
|
||||
revid = self._dostore(oid, data=MinPO(11))
|
||||
revid = self._dostore(oid, revid=revid, data=MinPO(12))
|
||||
# Undo the last transaction
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]['id'], [oid])
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(11))
|
||||
# Now from here, we can either redo the last undo, or undo the object
|
||||
# creation. Let's redo the last undo
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]['id'], [oid])
|
||||
data, revid = load_current(self._storage, oid)
|
||||
eq(zodb_unpickle(data), MinPO(12))
|
||||
self._iterate()
|
||||
|
||||
def checkTwoObjectUndo(self):
|
||||
eq = self.assertEqual
|
||||
# Convenience
|
||||
p31, p32, p51, p52 = map(zodb_pickle,
|
||||
map(MinPO, (31, 32, 51, 52)))
|
||||
oid1 = self._storage.new_oid()
|
||||
oid2 = self._storage.new_oid()
|
||||
revid1 = revid2 = ZERO
|
||||
# Store two objects in the same transaction
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid1, revid1, p31, '', t)
|
||||
self._storage.store(oid2, revid2, p51, '', t)
|
||||
# Finish the transaction
|
||||
self._storage.tpc_vote(t)
|
||||
tid = self._storage.tpc_finish(t)
|
||||
# Update those same two objects
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid1, tid, p32, '', t)
|
||||
self._storage.store(oid2, tid, p52, '', t)
|
||||
# Finish the transaction
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
# Make sure the objects have the current value
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(32))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(52))
|
||||
|
||||
# Now attempt to undo the transaction containing two objects
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]['id'], [oid1, oid2])
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(31))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(51))
|
||||
self._iterate()
|
||||
|
||||
def checkTwoObjectUndoAtOnce(self):
|
||||
# Convenience
|
||||
eq = self.assertEqual
|
||||
unless = self.assertTrue
|
||||
p30, p31, p32, p50, p51, p52 = map(zodb_pickle,
|
||||
map(MinPO,
|
||||
(30, 31, 32, 50, 51, 52)))
|
||||
oid1 = self._storage.new_oid()
|
||||
oid2 = self._storage.new_oid()
|
||||
# Store two objects in the same transaction
|
||||
tid = self._multi_obj_transaction([(oid1, ZERO, p30),
|
||||
(oid2, ZERO, p50),
|
||||
])
|
||||
# Update those same two objects
|
||||
tid = self._multi_obj_transaction([(oid1, tid, p31),
|
||||
(oid2, tid, p51),
|
||||
])
|
||||
# Update those same two objects
|
||||
tid = self._multi_obj_transaction([(oid1, tid, p32),
|
||||
(oid2, tid, p52),
|
||||
])
|
||||
# Make sure the objects have the current value
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(32))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(52))
|
||||
# Now attempt to undo the transaction containing two objects
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[0]['id']
|
||||
tid1 = info[1]['id']
|
||||
t = TransactionMetaData()
|
||||
oids = self._begin_undos_vote(t, tid, tid1)
|
||||
serial = self._storage.tpc_finish(t)
|
||||
# We may get the finalization stuff called an extra time,
|
||||
# depending on the implementation.
|
||||
if serial is None:
|
||||
self.assertEqual(oids, {oid1, oid2})
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(30))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(50))
|
||||
|
||||
# Now try to undo the one we just did to undo, whew
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]['id'], [oid1, oid2])
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(32))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(52))
|
||||
self._iterate()
|
||||
|
||||
def checkTwoObjectUndoAgain(self):
|
||||
eq = self.assertEqual
|
||||
p31, p32, p33, p51, p52, p53 = map(
|
||||
zodb_pickle,
|
||||
map(MinPO, (31, 32, 33, 51, 52, 53)))
|
||||
# Like the above, but the first revision of the objects are stored in
|
||||
# different transactions.
|
||||
oid1 = self._storage.new_oid()
|
||||
oid2 = self._storage.new_oid()
|
||||
revid1 = self._dostore(oid1, data=p31, already_pickled=1)
|
||||
revid2 = self._dostore(oid2, data=p51, already_pickled=1)
|
||||
# Update those same two objects
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid1, revid1, p32, '', t)
|
||||
self._storage.store(oid2, revid2, p52, '', t)
|
||||
# Finish the transaction
|
||||
self._storage.tpc_vote(t)
|
||||
self._storage.tpc_finish(t)
|
||||
# Now attempt to undo the transaction containing two objects
|
||||
info = self._storage.undoInfo()
|
||||
self._undo(info[0]["id"], [oid1, oid2])
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(31))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(51))
|
||||
# Like the above, but this time, the second transaction contains only
|
||||
# one object.
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid1, revid1, p33, '', t)
|
||||
self._storage.store(oid2, revid2, p53, '', t)
|
||||
# Finish the transaction
|
||||
self._storage.tpc_vote(t)
|
||||
tid = self._storage.tpc_finish(t)
|
||||
# Update in different transactions
|
||||
revid1 = self._dostore(oid1, revid=tid, data=MinPO(34))
|
||||
revid2 = self._dostore(oid2, revid=tid, data=MinPO(54))
|
||||
# Now attempt to undo the transaction containing two objects
|
||||
info = self._storage.undoInfo()
|
||||
self.undo(info[1]['id'])
|
||||
data, revid1 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(33))
|
||||
data, revid2 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(54))
|
||||
self._iterate()
|
||||
|
||||
def checkNotUndoable(self):
|
||||
eq = self.assertEqual
|
||||
# Set things up so we've got a transaction that can't be undone
|
||||
oid = self._storage.new_oid()
|
||||
revid_a = self._dostore(oid, data=MinPO(51))
|
||||
revid_b = self._dostore(oid, revid=revid_a, data=MinPO(52))
|
||||
revid_c = self._dostore(oid, revid=revid_b, data=MinPO(53))
|
||||
# Start the undo
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[1]['id']
|
||||
t = TransactionMetaData()
|
||||
self.assertRaises(POSException.UndoError,
|
||||
self._begin_undos_vote, t, tid)
|
||||
self._storage.tpc_abort(t)
|
||||
# Now have more fun: object1 and object2 are in the same transaction,
|
||||
# which we'll try to undo to, but one of them has since modified in
|
||||
# different transaction, so the undo should fail.
|
||||
oid1 = oid
|
||||
revid1 = revid_c
|
||||
oid2 = self._storage.new_oid()
|
||||
revid2 = ZERO
|
||||
p81, p82, p91, p92 = map(zodb_pickle,
|
||||
map(MinPO, (81, 82, 91, 92)))
|
||||
|
||||
t = TransactionMetaData()
|
||||
self._storage.tpc_begin(t)
|
||||
self._storage.store(oid1, revid1, p81, '', t)
|
||||
self._storage.store(oid2, revid2, p91, '', t)
|
||||
self._storage.tpc_vote(t)
|
||||
tid = self._storage.tpc_finish(t)
|
||||
# Make sure the objects have the expected values
|
||||
data, revid_11 = load_current(self._storage, oid1)
|
||||
eq(zodb_unpickle(data), MinPO(81))
|
||||
data, revid_22 = load_current(self._storage, oid2)
|
||||
eq(zodb_unpickle(data), MinPO(91))
|
||||
eq(revid_11, tid)
|
||||
eq(revid_22, tid)
|
||||
# Now modify oid2
|
||||
revid2 = self._dostore(oid2, tid, MinPO(92))
|
||||
self.assertNotEqual(tid, revid2)
|
||||
info = self._storage.undoInfo()
|
||||
tid = info[1]['id']
|
||||
t = TransactionMetaData()
|
||||
self.assertRaises(POSException.UndoError,
|
||||
self._begin_undos_vote, t, tid)
|
||||
self._storage.tpc_abort(t)
|
||||
self._iterate()
|
||||
|
||||
def checkTransactionalUndoAfterPack(self):
|
||||
# bwarsaw Date: Thu Mar 28 21:04:43 2002 UTC
|
||||
# This is a test which should provoke the underlying bug in
|
||||
# transactionalUndo() on a standby storage. If our hypothesis
|
||||
# is correct, the bug is in FileStorage, and is caused by
|
||||
# encoding the file position in the `id' field of the undoLog
|
||||
# information. Note that Full just encodes the tid, but this
|
||||
# is a problem for FileStorage (we have a strategy for fixing
|
||||
# this).
|
||||
|
||||
# So, basically, this makes sure that undo info doesn't depend
|
||||
# on file positions. We change the file positions in an undo
|
||||
# record by packing.
|
||||
|
||||
# Add a few object revisions
|
||||
oid = b'\0'*8
|
||||
revid0 = self._dostore(oid, data=MinPO(50))
|
||||
revid1 = self._dostore(oid, revid=revid0, data=MinPO(51))
|
||||
snooze()
|
||||
packtime = time.time()
|
||||
snooze() # time.time() now distinct from packtime
|
||||
revid2 = self._dostore(oid, revid=revid1, data=MinPO(52))
|
||||
self._dostore(oid, revid=revid2, data=MinPO(53))
|
||||
# Now get the undo log
|
||||
info = self._storage.undoInfo()
|
||||
self.assertEqual(len(info), 4)
|
||||
tid = info[0]['id']
|
||||
# Now pack just the initial revision of the object. We need the
|
||||
# second revision otherwise we won't be able to undo the third
|
||||
# revision!
|
||||
self._storage.pack(packtime, referencesf)
|
||||
# Make some basic assertions about the undo information now
|
||||
info2 = self._storage.undoInfo()
|
||||
self.assertEqual(len(info2), 2)
|
||||
# And now attempt to undo the last transaction
|
||||
undone, = self.undo(tid)
|
||||
self.assertEqual(undone, oid)
|
||||
data, revid = load_current(self._storage, oid)
|
||||
# The object must now be at the second state
|
||||
self.assertEqual(zodb_unpickle(data), MinPO(52))
|
||||
self._iterate()
|
||||
|
||||
def checkTransactionalUndoAfterPackWithObjectUnlinkFromRoot(self):
|
||||
eq = self.assertEqual
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
try:
|
||||
root = conn.root()
|
||||
|
||||
o1 = C()
|
||||
o2 = C()
|
||||
root['obj'] = o1
|
||||
o1.obj = o2
|
||||
txn = transaction.get()
|
||||
txn.note(u'o1 -> o2')
|
||||
txn.commit()
|
||||
now = packtime = time.time()
|
||||
while packtime <= now:
|
||||
packtime = time.time()
|
||||
|
||||
o3 = C()
|
||||
o2.obj = o3
|
||||
txn = transaction.get()
|
||||
txn.note(u'o1 -> o2 -> o3')
|
||||
txn.commit()
|
||||
|
||||
o1.obj = o3
|
||||
txn = transaction.get()
|
||||
txn.note(u'o1 -> o3')
|
||||
txn.commit()
|
||||
|
||||
log = self._storage.undoLog()
|
||||
eq(len(log), 4)
|
||||
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3',
|
||||
b'o1 -> o2', b'initial database creation')):
|
||||
eq(entry[0]['description'], entry[1])
|
||||
|
||||
self._storage.pack(packtime, referencesf)
|
||||
|
||||
log = self._storage.undoLog()
|
||||
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3')):
|
||||
eq(entry[0]['description'], entry[1])
|
||||
|
||||
tid = log[0]['id']
|
||||
db.undo(tid)
|
||||
txn = transaction.get()
|
||||
txn.note(u'undo')
|
||||
txn.commit()
|
||||
# undo does a txn-undo, but doesn't invalidate
|
||||
conn.sync()
|
||||
|
||||
log = self._storage.undoLog()
|
||||
for entry in zip(log, (b'undo', b'o1 -> o3', b'o1 -> o2 -> o3')):
|
||||
eq(entry[0]['description'], entry[1])
|
||||
|
||||
eq(o1.obj, o2)
|
||||
eq(o1.obj.obj, o3)
|
||||
self._iterate()
|
||||
finally:
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
def checkPackAfterUndoDeletion(self):
|
||||
db = DB(self._storage)
|
||||
cn = db.open()
|
||||
try:
|
||||
root = cn.root()
|
||||
|
||||
pack_times = []
|
||||
def set_pack_time():
|
||||
pack_times.append(time.time())
|
||||
snooze()
|
||||
|
||||
root["key0"] = MinPO(0)
|
||||
root["key1"] = MinPO(1)
|
||||
root["key2"] = MinPO(2)
|
||||
txn = transaction.get()
|
||||
txn.note(u"create 3 keys")
|
||||
txn.commit()
|
||||
|
||||
set_pack_time()
|
||||
|
||||
del root["key1"]
|
||||
txn = transaction.get()
|
||||
txn.note(u"delete 1 key")
|
||||
txn.commit()
|
||||
|
||||
set_pack_time()
|
||||
|
||||
root._p_deactivate()
|
||||
cn.sync()
|
||||
self.assertTrue(listeq(root.keys(), ["key0", "key2"]))
|
||||
|
||||
L = db.undoInfo()
|
||||
db.undo(L[0]["id"])
|
||||
txn = transaction.get()
|
||||
txn.note(u"undo deletion")
|
||||
txn.commit()
|
||||
|
||||
set_pack_time()
|
||||
|
||||
root._p_deactivate()
|
||||
cn.sync()
|
||||
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
|
||||
|
||||
for t in pack_times:
|
||||
self._storage.pack(t, referencesf)
|
||||
|
||||
root._p_deactivate()
|
||||
cn.sync()
|
||||
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
|
||||
for i in range(3):
|
||||
obj = root["key%d" % i]
|
||||
self.assertEqual(obj.value, i)
|
||||
root.items()
|
||||
self._inter_pack_pause()
|
||||
finally:
|
||||
cn.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def checkPackAfterUndoManyTimes(self):
|
||||
db = DB(self._storage)
|
||||
cn = db.open()
|
||||
try:
|
||||
rt = cn.root()
|
||||
|
||||
rt["test"] = MinPO(1)
|
||||
transaction.commit()
|
||||
rt["test2"] = MinPO(2)
|
||||
transaction.commit()
|
||||
rt["test"] = MinPO(3)
|
||||
txn = transaction.get()
|
||||
txn.note(u"root of undo")
|
||||
txn.commit()
|
||||
|
||||
packtimes = []
|
||||
for i in range(10):
|
||||
L = db.undoInfo()
|
||||
db.undo(L[0]["id"])
|
||||
txn = transaction.get()
|
||||
txn.note(u"undo %d" % i)
|
||||
txn.commit()
|
||||
rt._p_deactivate()
|
||||
cn.sync()
|
||||
|
||||
self.assertEqual(rt["test"].value, i % 2 and 3 or 1)
|
||||
self.assertEqual(rt["test2"].value, 2)
|
||||
|
||||
packtimes.append(time.time())
|
||||
snooze()
|
||||
|
||||
for t in packtimes:
|
||||
self._storage.pack(t, referencesf)
|
||||
cn.sync()
|
||||
|
||||
# TODO: Is _cache supposed to have a clear() method, or not?
|
||||
# cn._cache.clear()
|
||||
|
||||
# The last undo set the value to 3 and pack should
|
||||
# never change that.
|
||||
self.assertEqual(rt["test"].value, 3)
|
||||
self.assertEqual(rt["test2"].value, 2)
|
||||
self._inter_pack_pause()
|
||||
finally:
|
||||
cn.close()
|
||||
db.close()
|
||||
|
||||
def _inter_pack_pause(self):
|
||||
# DirectoryStorage needs a pause between packs,
|
||||
# most other storages dont.
|
||||
pass
|
||||
|
||||
def checkTransactionalUndoIterator(self):
|
||||
# check that data_txn set in iterator makes sense
|
||||
if not hasattr(self._storage, "iterator"):
|
||||
return
|
||||
|
||||
s = self._storage
|
||||
|
||||
BATCHES = 4
|
||||
OBJECTS = 4
|
||||
|
||||
orig = []
|
||||
for i in range(BATCHES):
|
||||
t = TransactionMetaData()
|
||||
tid = p64(i + 1)
|
||||
s.tpc_begin(t, tid)
|
||||
for j in range(OBJECTS):
|
||||
oid = s.new_oid()
|
||||
obj = MinPO(i * OBJECTS + j)
|
||||
s.store(oid, None, zodb_pickle(obj), '', t)
|
||||
orig.append((tid, oid))
|
||||
s.tpc_vote(t)
|
||||
s.tpc_finish(t)
|
||||
|
||||
orig = [(tid, oid, s.getTid(oid)) for tid, oid in orig]
|
||||
|
||||
i = 0
|
||||
for tid, oid, revid in orig:
|
||||
self._dostore(oid, revid=revid, data=MinPO(revid),
|
||||
description="update %s" % i)
|
||||
|
||||
# Undo the OBJECTS transactions that modified objects created
|
||||
# in the ith original transaction.
|
||||
|
||||
def undo(i):
|
||||
info = s.undoInfo()
|
||||
t = TransactionMetaData()
|
||||
s.tpc_begin(t)
|
||||
base = i * OBJECTS + i
|
||||
for j in range(OBJECTS):
|
||||
tid = info[base + j]['id']
|
||||
s.undo(tid, t)
|
||||
s.tpc_vote(t)
|
||||
s.tpc_finish(t)
|
||||
|
||||
for i in range(BATCHES):
|
||||
undo(i)
|
||||
|
||||
# There are now (2 + OBJECTS) * BATCHES transactions:
|
||||
# BATCHES original transactions, followed by
|
||||
# OBJECTS * BATCHES modifications, followed by
|
||||
# BATCHES undos
|
||||
|
||||
transactions = s.iterator()
|
||||
eq = self.assertEqual
|
||||
|
||||
for i in range(BATCHES):
|
||||
txn = next(transactions)
|
||||
|
||||
tid = p64(i + 1)
|
||||
eq(txn.tid, tid)
|
||||
|
||||
L1 = {(rec.oid, rec.tid, rec.data_txn) for rec in txn}
|
||||
L2 = {(oid, revid, None) for _tid, oid, revid in orig
|
||||
if _tid == tid}
|
||||
|
||||
eq(L1, L2)
|
||||
|
||||
for i in range(BATCHES * OBJECTS):
|
||||
txn = next(transactions)
|
||||
eq(len([rec for rec in txn if rec.data_txn is None]), 1)
|
||||
|
||||
for i in range(BATCHES):
|
||||
txn = next(transactions)
|
||||
|
||||
# The undos are performed in reverse order.
|
||||
otid = p64(BATCHES - i)
|
||||
L1 = [(rec.oid, rec.data_txn) for rec in txn]
|
||||
L2 = [(oid, otid) for _tid, oid, revid in orig
|
||||
if _tid == otid]
|
||||
L1.sort()
|
||||
L2.sort()
|
||||
eq(L1, L2)
|
||||
|
||||
self.assertRaises(StopIteration, next, transactions)
|
||||
|
||||
def checkUndoLogMetadata(self):
|
||||
# test that the metadata is correct in the undo log
|
||||
t = transaction.get()
|
||||
t.note(u't1')
|
||||
t.setExtendedInfo('k2', 'this is transaction metadata')
|
||||
t.setUser(u'u3',path=u'p3')
|
||||
db = DB(self._storage)
|
||||
conn = db.open()
|
||||
try:
|
||||
root = conn.root()
|
||||
o1 = C()
|
||||
root['obj'] = o1
|
||||
txn = transaction.get()
|
||||
txn.commit()
|
||||
l = self._storage.undoLog()
|
||||
self.assertEqual(len(l),2)
|
||||
d = l[0]
|
||||
self.assertEqual(d['description'], b't1')
|
||||
self.assertEqual(d['k2'], 'this is transaction metadata')
|
||||
self.assertEqual(d['user_name'], b'p3 u3')
|
||||
finally:
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
# A common test body for index tests on undoInfo and undoLog. Before
|
||||
# ZODB 3.4, they always returned a wrong number of results (one too
|
||||
# few _or_ too many, depending on how they were called).
|
||||
def _exercise_info_indices(self, method_name):
|
||||
db = DB(self._storage)
|
||||
info_func = getattr(db, method_name)
|
||||
cn = db.open()
|
||||
rt = cn.root()
|
||||
|
||||
# Do some transactions.
|
||||
for key in "abcdefghijklmnopqrstuvwxyz":
|
||||
rt[key] = ord(key)
|
||||
transaction.commit()
|
||||
|
||||
# 26 letters = 26 transactions, + the hidden transaction to make
|
||||
# the root object, == 27 expected.
|
||||
allofem = info_func(0, 100000)
|
||||
self.assertEqual(len(allofem), 27)
|
||||
|
||||
# Asking for no more than 100000 should do the same.
|
||||
redundant = info_func(last=-1000000)
|
||||
self.assertEqual(allofem, redundant)
|
||||
|
||||
# By default, we should get only 20 back.
|
||||
default = info_func()
|
||||
self.assertEqual(len(default), 20)
|
||||
# And they should be the most recent 20.
|
||||
self.assertEqual(default, allofem[:20])
|
||||
|
||||
# If we ask for only one, we should get only the most recent.
|
||||
fresh = info_func(last=1)
|
||||
self.assertEqual(len(fresh), 1)
|
||||
self.assertEqual(fresh[0], allofem[0])
|
||||
|
||||
# Another way of asking for only the most recent.
|
||||
redundant = info_func(last=-1)
|
||||
self.assertEqual(fresh, redundant)
|
||||
|
||||
# Try a slice that doesn't start at 0.
|
||||
oddball = info_func(first=11, last=17)
|
||||
self.assertEqual(len(oddball), 17-11)
|
||||
self.assertEqual(oddball, allofem[11 : 11+len(oddball)])
|
||||
|
||||
# And another way to spell the same thing.
|
||||
redundant = info_func(first=11, last=-6)
|
||||
self.assertEqual(oddball, redundant)
|
||||
|
||||
cn.close()
|
||||
# Caution: don't close db; the framework does that. If you close
|
||||
# it here, the ZODB tests still work, but the ZRS RecoveryStorageTests
|
||||
# fail (closing the DB here in those tests closes the ZRS primary
|
||||
# before a ZRS secondary even starts, and then the latter can't
|
||||
# find a server to recover from).
|
||||
|
||||
def checkIndicesInUndoInfo(self):
|
||||
self._exercise_info_indices("undoInfo")
|
||||
|
||||
def checkIndicesInUndoLog(self):
|
||||
self._exercise_info_indices("undoLog")
|
||||
|
||||
def checkUndoMultipleConflictResolution(self, reverse=False):
|
||||
from .ConflictResolution import PCounter
|
||||
db = DB(self._storage)
|
||||
cn = db.open()
|
||||
try:
|
||||
cn.root.x = PCounter()
|
||||
transaction.commit()
|
||||
|
||||
for i in range(4):
|
||||
with db.transaction() as conn:
|
||||
conn.transaction_manager.get().note(
|
||||
(str if PY3 else unicode)(i))
|
||||
conn.root.x.inc()
|
||||
|
||||
ids = [l['id'] for l in db.undoLog(1, 3)]
|
||||
if reverse:
|
||||
ids.reverse()
|
||||
|
||||
db.undoMultiple(ids)
|
||||
transaction.commit()
|
||||
|
||||
self.assertEqual(cn.root.x._value, 2)
|
||||
finally:
|
||||
cn.close()
|
||||
db.close()
|
||||
|
||||
def checkUndoMultipleConflictResolutionReversed(self):
|
||||
self.checkUndoMultipleConflictResolution(True)
|
||||
@@ -0,0 +1 @@
|
||||
# Having this makes debugging better.
|
||||
@@ -0,0 +1,198 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2005 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
ZODB Blob support
|
||||
=================
|
||||
|
||||
You create a blob like this::
|
||||
|
||||
>>> from ZODB.blob import Blob
|
||||
>>> myblob = Blob()
|
||||
|
||||
A blob implements the IBlob interface::
|
||||
|
||||
>>> from ZODB.interfaces import IBlob
|
||||
>>> IBlob.providedBy(myblob)
|
||||
True
|
||||
|
||||
We can open a new blob file for reading, but it won't have any data::
|
||||
|
||||
>>> with myblob.open("r") as fp: fp.read()
|
||||
''
|
||||
|
||||
But we can write data to a new Blob by opening it for writing::
|
||||
|
||||
>>> f = myblob.open("w")
|
||||
>>> _ = f.write(b"Hi, Blob!")
|
||||
|
||||
If we try to open a Blob again while it is open for writing, we get an error::
|
||||
|
||||
>>> myblob.open("r")
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Already opened for writing.
|
||||
|
||||
We can close the file::
|
||||
|
||||
>>> f.close()
|
||||
|
||||
Now we can open it for reading::
|
||||
|
||||
>>> f2 = myblob.open("r")
|
||||
|
||||
And we get the data back::
|
||||
|
||||
>>> f2.read()
|
||||
'Hi, Blob!'
|
||||
|
||||
If we want to, we can open it again::
|
||||
|
||||
>>> f3 = myblob.open("r")
|
||||
>>> f3.read()
|
||||
'Hi, Blob!'
|
||||
|
||||
But we can't open it for writing, while it is opened for reading::
|
||||
|
||||
>>> myblob.open("a")
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Already opened for reading.
|
||||
|
||||
Before we can write, we have to close the readers::
|
||||
|
||||
>>> f2.close()
|
||||
>>> f3.close()
|
||||
|
||||
Now we can open it for writing again and e.g. append data::
|
||||
|
||||
>>> f4 = myblob.open("a")
|
||||
>>> _ = f4.write(b"\nBlob is fine.")
|
||||
|
||||
We can't open a blob while it is open for writing:
|
||||
|
||||
>>> myblob.open("w")
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Already opened for writing.
|
||||
|
||||
>>> myblob.open("r")
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Already opened for writing.
|
||||
|
||||
>>> f4.close()
|
||||
|
||||
Now we can read it::
|
||||
|
||||
>>> f4a = myblob.open("r")
|
||||
>>> f4a.read()
|
||||
'Hi, Blob!\nBlob is fine.'
|
||||
>>> f4a.close()
|
||||
|
||||
You shouldn't need to explicitly close a blob unless you hold a reference
|
||||
to it via a name. If the first line in the following test kept a reference
|
||||
around via a name, the second call to open it in a writable mode would fail
|
||||
with a BlobError, but it doesn't::
|
||||
|
||||
>>> with myblob.open("r+") as fp: fp.read()
|
||||
'Hi, Blob!\nBlob is fine.'
|
||||
>>> f4b = myblob.open("a")
|
||||
>>> f4b.close()
|
||||
|
||||
We can read lines out of the blob too::
|
||||
|
||||
>>> f5 = myblob.open("r")
|
||||
>>> f5.readline()
|
||||
'Hi, Blob!\n'
|
||||
>>> f5.readline()
|
||||
'Blob is fine.'
|
||||
>>> f5.close()
|
||||
|
||||
We can seek to certain positions in a blob and read portions of it::
|
||||
|
||||
>>> f6 = myblob.open('r')
|
||||
>>> _ = f6.seek(4)
|
||||
>>> int(f6.tell())
|
||||
4
|
||||
>>> f6.read(5)
|
||||
'Blob!'
|
||||
>>> f6.close()
|
||||
|
||||
We can use the object returned by a blob open call as an iterable::
|
||||
|
||||
>>> f7 = myblob.open('r')
|
||||
>>> for line in f7:
|
||||
... print(line.decode())
|
||||
Hi, Blob!
|
||||
<BLANKLINE>
|
||||
Blob is fine.
|
||||
>>> f7.close()
|
||||
|
||||
We can truncate a blob::
|
||||
|
||||
>>> f8 = myblob.open('a')
|
||||
>>> _ = f8.truncate(0)
|
||||
>>> f8.close()
|
||||
>>> f8 = myblob.open('r')
|
||||
>>> f8.read()
|
||||
''
|
||||
>>> f8.close()
|
||||
|
||||
Blobs are always opened in binary mode::
|
||||
|
||||
>>> f9 = myblob.open("r")
|
||||
>>> f9.mode
|
||||
'rb'
|
||||
>>> f9.close()
|
||||
|
||||
Blobs that have not been committed can be opened using any mode,
|
||||
except for "c"::
|
||||
|
||||
>>> import six
|
||||
>>> from ZODB.blob import BlobError, valid_modes
|
||||
>>> for mode in valid_modes:
|
||||
... try:
|
||||
... f10 = Blob().open(mode)
|
||||
... except BlobError:
|
||||
... six.print_('open failed with mode "%s"' % mode)
|
||||
... else:
|
||||
... f10.close()
|
||||
open failed with mode "c"
|
||||
|
||||
Some cleanup in this test is needed::
|
||||
|
||||
>>> import transaction
|
||||
>>> transaction.get().abort()
|
||||
|
||||
Subclassing Blobs
|
||||
-----------------
|
||||
|
||||
Blobs are not subclassable::
|
||||
|
||||
>>> class SubBlob(Blob):
|
||||
... pass
|
||||
>>> my_sub_blob = SubBlob()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
TypeError: Blobs do not support subclassing.
|
||||
|
||||
Passing data to the blob constructor
|
||||
------------------------------------
|
||||
|
||||
If you have a small amount of data, you can pass it to the blob
|
||||
constructor. (This is a convenience, mostly for writing tests.)
|
||||
|
||||
>>> myblob = Blob(b'some data')
|
||||
>>> with myblob.open() as fp: fp.read()
|
||||
'some data'
|
||||
@@ -0,0 +1,88 @@
|
||||
Connection support for Blobs tests
|
||||
==================================
|
||||
|
||||
Connections handle Blobs specially. To demonstrate that, we first need a Blob
|
||||
with some data:
|
||||
|
||||
>>> from ZODB.interfaces import IBlob
|
||||
>>> from ZODB.blob import Blob
|
||||
>>> import transaction
|
||||
>>> blob = Blob()
|
||||
>>> data = blob.open("w")
|
||||
>>> _ = data.write(b"I'm a happy Blob.")
|
||||
>>> data.close()
|
||||
|
||||
We also need a database with a blob supporting storage. (We're going to use
|
||||
FileStorage rather than MappingStorage here because we will want ``loadBefore``
|
||||
for one of our examples.)
|
||||
|
||||
>>> blob_storage = create_storage()
|
||||
>>> from ZODB.DB import DB
|
||||
>>> database = DB(blob_storage)
|
||||
|
||||
Putting a Blob into a Connection works like every other object:
|
||||
|
||||
>>> connection = database.open()
|
||||
>>> root = connection.root()
|
||||
>>> root['myblob'] = blob
|
||||
>>> transaction.commit()
|
||||
|
||||
We can also commit a transaction that seats a blob into place without
|
||||
calling the blob's open method:
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> anotherblob = Blob()
|
||||
>>> root['anotherblob'] = anotherblob
|
||||
>>> nothing = transaction.commit()
|
||||
|
||||
Getting stuff out of there works similarly:
|
||||
|
||||
>>> transaction2 = transaction.TransactionManager()
|
||||
>>> connection2 = database.open(transaction_manager=transaction2)
|
||||
>>> root = connection2.root()
|
||||
>>> blob2 = root['myblob']
|
||||
>>> IBlob.providedBy(blob2)
|
||||
True
|
||||
>>> with blob2.open("r") as fp: fp.read()
|
||||
"I'm a happy Blob."
|
||||
>>> transaction2.abort()
|
||||
|
||||
MVCC also works.
|
||||
|
||||
>>> transaction3 = transaction.TransactionManager()
|
||||
>>> connection3 = database.open(transaction_manager=transaction3)
|
||||
>>> f = connection.root()['myblob'].open('w')
|
||||
>>> _ = f.write(b'I am an ecstatic Blob.')
|
||||
>>> f.close()
|
||||
>>> transaction.commit()
|
||||
>>> with connection3.root()['myblob'].open('r') as fp: fp.read()
|
||||
"I'm a happy Blob."
|
||||
|
||||
>>> transaction2.abort()
|
||||
>>> transaction3.abort()
|
||||
>>> connection2.close()
|
||||
>>> connection3.close()
|
||||
|
||||
You can't put blobs into a database that has uses a Non-Blob-Storage, though:
|
||||
|
||||
>>> from ZODB.MappingStorage import MappingStorage
|
||||
>>> no_blob_storage = MappingStorage()
|
||||
>>> database2 = DB(no_blob_storage)
|
||||
>>> connection2 = database2.open(transaction_manager=transaction2)
|
||||
>>> root = connection2.root()
|
||||
>>> root['myblob'] = Blob()
|
||||
>>> transaction2.commit() # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
Unsupported: Storing Blobs in ...
|
||||
|
||||
>>> transaction2.abort()
|
||||
>>> connection2.close()
|
||||
|
||||
After testing this, we don't need the storage directory and databases anymore:
|
||||
|
||||
>>> transaction.abort()
|
||||
>>> connection.close()
|
||||
>>> database.close()
|
||||
>>> database2.close()
|
||||
>>> blob_storage.close()
|
||||
@@ -0,0 +1,132 @@
|
||||
Consuming existing files
|
||||
========================
|
||||
|
||||
The ZODB Blob implementation allows to import existing files as Blobs within
|
||||
an O(1) operation we call `consume`::
|
||||
|
||||
Let's create a file::
|
||||
|
||||
>>> to_import = open('to_import', 'wb')
|
||||
>>> _ = to_import.write(b"I'm a Blob and I feel fine.")
|
||||
|
||||
The file *must* be closed before giving it to consumeFile:
|
||||
|
||||
>>> to_import.close()
|
||||
|
||||
Now, let's consume this file in a blob by specifying it's name::
|
||||
|
||||
>>> from ZODB.blob import Blob
|
||||
>>> blob = Blob()
|
||||
>>> blob.consumeFile('to_import')
|
||||
|
||||
After the consumeFile operation, the original file has been removed:
|
||||
|
||||
>>> import os
|
||||
>>> os.path.exists('to_import')
|
||||
False
|
||||
|
||||
We now can call open on the blob and read and write the data::
|
||||
|
||||
>>> blob_read = blob.open('r')
|
||||
>>> blob_read.read()
|
||||
"I'm a Blob and I feel fine."
|
||||
>>> blob_read.close()
|
||||
>>> blob_write = blob.open('w')
|
||||
>>> _ = blob_write.write(b'I was changed.')
|
||||
>>> blob_write.close()
|
||||
|
||||
We can not consume a file when there is a reader or writer around for a blob
|
||||
already::
|
||||
|
||||
>>> with open('to_import', 'wb') as file:
|
||||
... _ = file.write(b'I am another blob.')
|
||||
>>> blob_read = blob.open('r')
|
||||
>>> blob.consumeFile('to_import')
|
||||
Traceback (most recent call last):
|
||||
BlobError: Already opened for reading.
|
||||
>>> blob_read.close()
|
||||
>>> blob_write = blob.open('w')
|
||||
>>> blob.consumeFile('to_import')
|
||||
Traceback (most recent call last):
|
||||
BlobError: Already opened for writing.
|
||||
>>> blob_write.close()
|
||||
|
||||
Now, after closing all readers and writers we can consume files again::
|
||||
|
||||
>>> blob.consumeFile('to_import')
|
||||
>>> blob_read = blob.open('r')
|
||||
>>> blob_read.read()
|
||||
'I am another blob.'
|
||||
|
||||
>>> blob_read.close()
|
||||
|
||||
Edge cases
|
||||
==========
|
||||
|
||||
There are some edge cases what happens when the link() operation
|
||||
fails. We simulate this in different states:
|
||||
|
||||
Case 1: We don't have uncommitted data, but the link operation fails. We fall
|
||||
back to try a copy/remove operation that is successfull::
|
||||
|
||||
>>> with open('to_import', 'wb') as file:
|
||||
... _ = file.write(b'Some data.')
|
||||
|
||||
>>> def failing_rename(f1, f2):
|
||||
... if f1 == 'to_import':
|
||||
... raise OSError("I can't link.")
|
||||
... os_rename(f1, f2)
|
||||
|
||||
>>> blob = Blob()
|
||||
>>> os_rename = os.rename
|
||||
>>> os.rename = failing_rename
|
||||
>>> blob.consumeFile('to_import')
|
||||
|
||||
The blob did not have data before, so it shouldn't have data now::
|
||||
|
||||
>>> with blob.open('r') as fp: fp.read()
|
||||
'Some data.'
|
||||
|
||||
Case 2: We don't have uncommitted data and both the link operation and the
|
||||
copy fail. The exception will be re-raised and the target file will not
|
||||
exist::
|
||||
|
||||
>>> blob = Blob()
|
||||
>>> import ZODB.utils
|
||||
>>> utils_cp = ZODB.utils.cp
|
||||
|
||||
>>> def failing_copy(f1, f2):
|
||||
... raise OSError("I can't copy.")
|
||||
|
||||
>>> ZODB.utils.cp = failing_copy
|
||||
>>> with open('to_import', 'wb') as file:
|
||||
... _ = file.write(b'Some data.')
|
||||
>>> blob.consumeFile('to_import')
|
||||
Traceback (most recent call last):
|
||||
OSError: I can't copy.
|
||||
|
||||
The blob did not have data before, so it shouldn't have data now::
|
||||
|
||||
>>> with blob.open('r') as fp: fp.read()
|
||||
''
|
||||
|
||||
Case 3: We have uncommitted data, but the link and the copy operations fail.
|
||||
The exception will be re-raised and the target file will exist with the
|
||||
previous uncomitted data::
|
||||
|
||||
>>> blob = Blob()
|
||||
>>> with blob.open('w') as blob_writing:
|
||||
... _ = blob_writing.write(b'Uncommitted data')
|
||||
|
||||
>>> blob.consumeFile('to_import')
|
||||
Traceback (most recent call last):
|
||||
OSError: I can't copy.
|
||||
|
||||
The blob did existed before and had uncommitted data, this shouldn't have
|
||||
changed::
|
||||
|
||||
>>> with blob.open('r') as fp: fp.read()
|
||||
'Uncommitted data'
|
||||
|
||||
>>> os.rename = os_rename
|
||||
>>> ZODB.utils.cp = utils_cp
|
||||
@@ -0,0 +1,70 @@
|
||||
Import/export support for blob data
|
||||
===================================
|
||||
|
||||
Set up:
|
||||
|
||||
>>> import ZODB.blob, transaction
|
||||
>>> from persistent.mapping import PersistentMapping
|
||||
|
||||
We need an database with an undoing blob supporting storage:
|
||||
|
||||
>>> database1 = ZODB.DB(create_storage('1'))
|
||||
>>> database2 = ZODB.DB(create_storage('2'))
|
||||
|
||||
Create our root object for database1:
|
||||
|
||||
>>> connection1 = database1.open()
|
||||
>>> root1 = connection1.root()
|
||||
|
||||
Put a couple blob objects in our database1 and on the filesystem:
|
||||
|
||||
>>> import time, os
|
||||
>>> nothing = transaction.begin()
|
||||
>>> data1 = b'x'*100000
|
||||
>>> blob1 = ZODB.blob.Blob()
|
||||
>>> with blob1.open('w') as file:
|
||||
... _ = file.write(data1)
|
||||
>>> data2 = b'y'*100000
|
||||
>>> blob2 = ZODB.blob.Blob()
|
||||
>>> with blob2.open('w') as file:
|
||||
... _ = file.write(data2)
|
||||
>>> d = PersistentMapping({'blob1':blob1, 'blob2':blob2})
|
||||
>>> root1['blobdata'] = d
|
||||
>>> transaction.commit()
|
||||
|
||||
Export our blobs from a database1 connection:
|
||||
|
||||
>>> conn = root1['blobdata']._p_jar
|
||||
>>> oid = root1['blobdata']._p_oid
|
||||
>>> exportfile = 'export'
|
||||
>>> connection1.exportFile(oid, exportfile).close()
|
||||
|
||||
Import our exported data into database2:
|
||||
|
||||
>>> connection2 = database2.open()
|
||||
>>> root2 = connection2.root()
|
||||
>>> nothing = transaction.begin()
|
||||
>>> data = root2._p_jar.importFile(exportfile)
|
||||
>>> root2['blobdata'] = data
|
||||
>>> transaction.commit()
|
||||
|
||||
Make sure our data exists:
|
||||
|
||||
>>> items1 = root1['blobdata']
|
||||
>>> items2 = root2['blobdata']
|
||||
>>> bool(items1.keys() == items2.keys())
|
||||
True
|
||||
>>> with items1['blob1'].open() as fp1:
|
||||
... with items2['blob1'].open() as fp2:
|
||||
... fp1.read() == fp2.read()
|
||||
True
|
||||
>>> with items1['blob2'].open() as fp1:
|
||||
... with items2['blob2'].open() as fp2:
|
||||
... fp1.read() == fp2.read()
|
||||
True
|
||||
>>> transaction.get().abort()
|
||||
|
||||
.. cleanup
|
||||
|
||||
>>> database1.close()
|
||||
>>> database2.close()
|
||||
@@ -0,0 +1,303 @@
|
||||
======================
|
||||
Blob directory layouts
|
||||
======================
|
||||
|
||||
The internal structure of the blob directories is governed by so called
|
||||
`layouts`. The current default layout is called `bushy`.
|
||||
|
||||
The original blob implementation used a layout that we now call `lawn` and
|
||||
which is still available for backwards compatibility.
|
||||
|
||||
Layouts implement two methods: one for computing a relative path for an
|
||||
OID and one for turning a relative path back into an OID.
|
||||
|
||||
Our terminology is roughly the same as used in `DirectoryStorage`.
|
||||
|
||||
The `bushy` layout
|
||||
==================
|
||||
|
||||
The bushy layout splits the OID into the 8 byte parts, reverses them and
|
||||
creates one directory level for each part, named by the hexlified
|
||||
representation of the byte value. This results in 8 levels of directories, the
|
||||
leaf directories being used for the revisions of the blobs and at most 256
|
||||
entries per directory level:
|
||||
|
||||
>>> from ZODB.blob import BushyLayout
|
||||
>>> bushy = BushyLayout()
|
||||
>>> bushy.oid_to_path(b'\x00\x00\x00\x00\x00\x00\x00\x00')
|
||||
'0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x00'
|
||||
>>> bushy.oid_to_path(b'\x00\x00\x00\x00\x00\x00\x00\x01')
|
||||
'0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x01'
|
||||
|
||||
>>> import os
|
||||
>>> bushy.path_to_oid(os.path.join(
|
||||
... '0x01', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00'))
|
||||
'\x01\x00\x00\x00\x00\x00\x00\x00'
|
||||
>>> bushy.path_to_oid(os.path.join(
|
||||
... '0xff', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00', '0x00'))
|
||||
'\xff\x00\x00\x00\x00\x00\x00\x00'
|
||||
|
||||
Paths that do not represent an OID will cause a ValueError:
|
||||
|
||||
>>> bushy.path_to_oid('tmp')
|
||||
Traceback (most recent call last):
|
||||
ValueError: Not a valid OID path: `tmp`
|
||||
|
||||
|
||||
The `lawn` layout
|
||||
=================
|
||||
|
||||
The lawn layout creates on directory for each blob named by the blob's hex
|
||||
representation of its OID. This has some limitations on various file systems
|
||||
like performance penalties or the inability to store more than a given number
|
||||
of blobs at the same time (e.g. 32k on ext3).
|
||||
|
||||
>>> from ZODB.blob import LawnLayout
|
||||
>>> lawn = LawnLayout()
|
||||
>>> lawn.oid_to_path(b'\x00\x00\x00\x00\x00\x00\x00\x00')
|
||||
'0x00'
|
||||
>>> lawn.oid_to_path(b'\x00\x00\x00\x00\x00\x00\x00\x01')
|
||||
'0x01'
|
||||
|
||||
>>> lawn.path_to_oid('0x01')
|
||||
'\x00\x00\x00\x00\x00\x00\x00\x01'
|
||||
|
||||
Paths that do not represent an OID will cause a ValueError:
|
||||
|
||||
>>> lawn.path_to_oid('tmp')
|
||||
Traceback (most recent call last):
|
||||
ValueError: Not a valid OID path: `tmp`
|
||||
>>> lawn.path_to_oid('')
|
||||
Traceback (most recent call last):
|
||||
ValueError: Not a valid OID path: ``
|
||||
|
||||
|
||||
Auto-detecting the layout of a directory
|
||||
========================================
|
||||
|
||||
To allow easier migration, we provide an auto-detection feature that analyses a
|
||||
blob directory and decides for a strategy to use. In general it prefers to
|
||||
choose the `bushy` layout, except if it determines that the directory has
|
||||
already been used to create a lawn structure.
|
||||
|
||||
>>> from ZODB.blob import auto_layout_select
|
||||
|
||||
1. Non-existing directories will trigger a bushy layout:
|
||||
|
||||
>>> import os, shutil
|
||||
>>> auto_layout_select('blobs')
|
||||
'bushy'
|
||||
|
||||
2. Empty directories will trigger a bushy layout too:
|
||||
|
||||
>>> os.mkdir('blobs')
|
||||
>>> auto_layout_select('blobs')
|
||||
'bushy'
|
||||
|
||||
3. If the directory contains a marker for the strategy it will be used:
|
||||
|
||||
>>> from ZODB.blob import LAYOUT_MARKER
|
||||
>>> import os.path
|
||||
>>> with open(os.path.join('blobs', LAYOUT_MARKER), 'wb') as file:
|
||||
... _ = file.write(b'bushy')
|
||||
>>> auto_layout_select('blobs')
|
||||
'bushy'
|
||||
>>> with open(os.path.join('blobs', LAYOUT_MARKER), 'wb') as file:
|
||||
... _ = file.write(b'lawn')
|
||||
>>> auto_layout_select('blobs')
|
||||
'lawn'
|
||||
>>> shutil.rmtree('blobs')
|
||||
|
||||
4. If the directory does not contain a marker but other files that are
|
||||
not hidden, we assume that it was created with an earlier version of
|
||||
the blob implementation and uses our `lawn` layout:
|
||||
|
||||
>>> os.mkdir('blobs')
|
||||
>>> with open(os.path.join('blobs', '0x0101'), 'wb') as file:
|
||||
... _ = file.write(b'foo')
|
||||
>>> auto_layout_select('blobs')
|
||||
'lawn'
|
||||
>>> shutil.rmtree('blobs')
|
||||
|
||||
5. If the directory contains only hidden files, use the bushy layout:
|
||||
|
||||
>>> os.mkdir('blobs')
|
||||
>>> with open(os.path.join('blobs', '.svn'), 'wb') as file:
|
||||
... _ = file.write(b'blah')
|
||||
>>> auto_layout_select('blobs')
|
||||
'bushy'
|
||||
>>> shutil.rmtree('blobs')
|
||||
|
||||
|
||||
Directory layout markers
|
||||
========================
|
||||
|
||||
When the file system helper (FSH) is asked to create the directory structure,
|
||||
it will leave a marker with the choosen layout if no marker exists yet:
|
||||
|
||||
>>> from ZODB.blob import FilesystemHelper
|
||||
>>> blobs = 'blobs'
|
||||
>>> fsh = FilesystemHelper(blobs)
|
||||
>>> fsh.layout_name
|
||||
'bushy'
|
||||
>>> fsh.create()
|
||||
>>> with open(os.path.join(blobs, LAYOUT_MARKER), 'rb') as fp: fp.read()
|
||||
'bushy'
|
||||
|
||||
If the FSH finds a marker, then it verifies whether its content matches the
|
||||
strategy that was chosen. It will raise an exception if we try to work with a
|
||||
directory that has a different marker than the chosen strategy:
|
||||
|
||||
>>> fsh = FilesystemHelper(blobs, 'lawn')
|
||||
>>> fsh.layout_name
|
||||
'lawn'
|
||||
>>> fsh.create() # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
ValueError: Directory layout `lawn` selected for blob directory .../blobs/, but marker found for layout `bushy`
|
||||
>>> rmtree(blobs)
|
||||
|
||||
This function interacts with the automatic detection in the way, that an
|
||||
unmarked directory will be marked the first time when it is auto-guessed and
|
||||
the marker will be used in the future:
|
||||
|
||||
>>> import ZODB.FileStorage
|
||||
>>> from ZODB.blob import BlobStorage
|
||||
>>> datafs = 'data.fs'
|
||||
>>> base_storage = ZODB.FileStorage.FileStorage(datafs)
|
||||
|
||||
>>> os.mkdir(blobs)
|
||||
>>> with open(os.path.join(blobs, 'foo'), 'wb') as file:
|
||||
... _ = file.write(b'foo')
|
||||
>>> blob_storage = BlobStorage(blobs, base_storage)
|
||||
>>> blob_storage.fshelper.layout_name
|
||||
'lawn'
|
||||
>>> with open(os.path.join(blobs, LAYOUT_MARKER), 'rb') as fp: fp.read()
|
||||
'lawn'
|
||||
>>> blob_storage = BlobStorage('blobs', base_storage, layout='bushy')
|
||||
... # doctest: +ELLIPSIS
|
||||
Traceback (most recent call last):
|
||||
ValueError: Directory layout `bushy` selected for blob directory .../blobs/, but marker found for layout `lawn`
|
||||
|
||||
|
||||
>>> base_storage.close()
|
||||
>>> rmtree('blobs')
|
||||
|
||||
|
||||
Migrating between directory layouts
|
||||
===================================
|
||||
|
||||
A script called `migrateblobs.py` is distributed with the ZODB for offline
|
||||
migration capabilities between different directory layouts. It can migrate any
|
||||
blob directory layout to any other layout. It leaves the original blob
|
||||
directory untouched (except from eventually creating a temporary directory and
|
||||
the storage layout marker).
|
||||
|
||||
The migration is accessible as a library function:
|
||||
|
||||
>>> from ZODB.scripts.migrateblobs import migrate
|
||||
|
||||
Create a `lawn` directory structure and migrate it to the new `bushy` one:
|
||||
|
||||
>>> from ZODB.blob import FilesystemHelper
|
||||
>>> d = 'd'
|
||||
>>> os.mkdir(d)
|
||||
>>> old = os.path.join(d, 'old')
|
||||
>>> old_fsh = FilesystemHelper(old, 'lawn')
|
||||
>>> old_fsh.create()
|
||||
>>> blob1 = old_fsh.getPathForOID(7039, create=True)
|
||||
>>> blob2 = old_fsh.getPathForOID(10, create=True)
|
||||
>>> blob3 = old_fsh.getPathForOID(7034, create=True)
|
||||
>>> with open(os.path.join(blob1, 'foo'), 'wb') as file:
|
||||
... _ = file.write(b'foo')
|
||||
>>> with open(os.path.join(blob1, 'foo2'), 'wb') as file:
|
||||
... _ = file.write(b'bar')
|
||||
>>> with open(os.path.join(blob2, 'foo3'), 'wb') as file:
|
||||
... _ = file.write(b'baz')
|
||||
>>> with open(os.path.join(blob2, 'foo4'), 'wb') as file:
|
||||
... _ = file.write(b'qux')
|
||||
>>> with open(os.path.join(blob3, 'foo5'), 'wb') as file:
|
||||
... _ = file.write(b'quux')
|
||||
>>> with open(os.path.join(blob3, 'foo6'), 'wb') as file:
|
||||
... _ = file.write(b'corge')
|
||||
|
||||
Committed blobs have their permissions set to 000
|
||||
|
||||
The migration function is called with the old and the new path and the layout
|
||||
that shall be used for the new directory:
|
||||
|
||||
>>> bushy = os.path.join(d, 'bushy')
|
||||
>>> migrate(old, bushy, 'bushy') # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
||||
Migrating blob data from `.../old` (lawn) to `.../bushy` (bushy)
|
||||
OID: 0x0a - 2 files
|
||||
OID: 0x1b7a - 2 files
|
||||
OID: 0x1b7f - 2 files
|
||||
|
||||
The new directory now contains the same files in different directories, but
|
||||
with the same sizes and permissions:
|
||||
|
||||
>>> lawn_files = {}
|
||||
>>> for base, dirs, files in os.walk(old):
|
||||
... for file_name in files:
|
||||
... lawn_files[file_name] = os.path.join(base, file_name)
|
||||
|
||||
>>> bushy_files = {}
|
||||
>>> for base, dirs, files in os.walk(bushy):
|
||||
... for file_name in files:
|
||||
... bushy_files[file_name] = os.path.join(base, file_name)
|
||||
|
||||
>>> len(lawn_files) == len(bushy_files)
|
||||
True
|
||||
|
||||
>>> import six
|
||||
>>> for file_name, lawn_path in sorted(lawn_files.items()):
|
||||
... if file_name == '.layout':
|
||||
... continue
|
||||
... lawn_stat = os.stat(lawn_path)
|
||||
... bushy_path = bushy_files[file_name]
|
||||
... bushy_stat = os.stat(bushy_path)
|
||||
... six.print_(lawn_path, '-->', bushy_path)
|
||||
... if ((lawn_stat.st_mode, lawn_stat.st_size) !=
|
||||
... (bushy_stat.st_mode, bushy_stat.st_size)):
|
||||
... print('oops')
|
||||
old/0x1b7f/foo --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7f/foo
|
||||
old/0x1b7f/foo2 --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7f/foo2
|
||||
old/0x0a/foo3 --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x0a/foo3
|
||||
old/0x0a/foo4 --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x0a/foo4
|
||||
old/0x1b7a/foo5 --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo5
|
||||
old/0x1b7a/foo6 --> bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo6
|
||||
|
||||
We can also migrate the bushy layout back to the lawn layout:
|
||||
|
||||
>>> lawn = os.path.join(d, 'lawn')
|
||||
>>> migrate(bushy, lawn, 'lawn')
|
||||
Migrating blob data from `.../bushy` (bushy) to `.../lawn` (lawn)
|
||||
OID: 0x0a - 2 files
|
||||
OID: 0x1b7a - 2 files
|
||||
OID: 0x1b7f - 2 files
|
||||
|
||||
>>> lawn_files = {}
|
||||
>>> for base, dirs, files in os.walk(lawn):
|
||||
... for file_name in files:
|
||||
... lawn_files[file_name] = os.path.join(base, file_name)
|
||||
|
||||
>>> len(lawn_files) == len(bushy_files)
|
||||
True
|
||||
|
||||
>>> for file_name, lawn_path in sorted(lawn_files.items()):
|
||||
... if file_name == '.layout':
|
||||
... continue
|
||||
... lawn_stat = os.stat(lawn_path)
|
||||
... bushy_path = bushy_files[file_name]
|
||||
... bushy_stat = os.stat(bushy_path)
|
||||
... six.print_(bushy_path, '-->', lawn_path)
|
||||
... if ((lawn_stat.st_mode, lawn_stat.st_size) !=
|
||||
... (bushy_stat.st_mode, bushy_stat.st_size)):
|
||||
... print('oops')
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7f/foo --> lawn/0x1b7f/foo
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7f/foo2 --> lawn/0x1b7f/foo2
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x0a/foo3 --> lawn/0x0a/foo3
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x0a/foo4 --> lawn/0x0a/foo4
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo5 --> lawn/0x1b7a/foo5
|
||||
bushy/0x00/0x00/0x00/0x00/0x00/0x00/0x1b/0x7a/foo6 --> lawn/0x1b7a/foo6
|
||||
|
||||
>>> rmtree(d)
|
||||
@@ -0,0 +1,112 @@
|
||||
Packing support for blob data
|
||||
=============================
|
||||
|
||||
Set up:
|
||||
|
||||
>>> from ZODB.serialize import referencesf
|
||||
>>> from ZODB.blob import Blob
|
||||
>>> from ZODB import utils
|
||||
>>> from ZODB.DB import DB
|
||||
>>> import transaction
|
||||
|
||||
A helper method to assure a unique timestamp across multiple platforms:
|
||||
|
||||
>>> from ZODB.tests.testblob import new_time
|
||||
|
||||
UNDOING
|
||||
=======
|
||||
|
||||
We need a database with an undoing blob supporting storage:
|
||||
|
||||
>>> blob_storage = create_storage()
|
||||
>>> database = DB(blob_storage)
|
||||
|
||||
Create our root object:
|
||||
|
||||
>>> connection1 = database.open()
|
||||
>>> root = connection1.root()
|
||||
|
||||
Put some revisions of a blob object in our database and on the filesystem:
|
||||
|
||||
>>> import os
|
||||
>>> tids = []
|
||||
>>> times = []
|
||||
>>> blob = Blob()
|
||||
|
||||
>>> for i in range(5):
|
||||
... _ = transaction.begin()
|
||||
... times.append(new_time())
|
||||
... with blob.open('w') as file:
|
||||
... _ = file.write(b'this is blob data ' + str(i).encode())
|
||||
... if i:
|
||||
... tids.append(blob._p_serial)
|
||||
... else:
|
||||
... root['blob'] = blob
|
||||
... transaction.commit()
|
||||
|
||||
>>> blob._p_activate()
|
||||
>>> tids.append(blob._p_serial)
|
||||
|
||||
>>> oid = root['blob']._p_oid
|
||||
>>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
|
||||
>>> [ os.path.exists(x) for x in fns ] # no pack
|
||||
[True, True, True, True, True]
|
||||
|
||||
Do a pack to the slightly before the first revision was written:
|
||||
|
||||
>>> packtime = times[0]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[True, True, True, True, True]
|
||||
|
||||
Do a pack to the slightly before the second revision was written:
|
||||
|
||||
>>> packtime = times[1]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[True, True, True, True, True]
|
||||
|
||||
Do a pack to the slightly before the third revision was written:
|
||||
|
||||
>>> packtime = times[2]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, True, True, True, True]
|
||||
|
||||
Do a pack to the slightly before the fourth revision was written:
|
||||
|
||||
>>> packtime = times[3]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, True, True, True]
|
||||
|
||||
Do a pack to the slightly before the fifth revision was written:
|
||||
|
||||
>>> packtime = times[4]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, True, True]
|
||||
|
||||
Do a pack to now:
|
||||
|
||||
>>> packtime = new_time()
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, False, True]
|
||||
|
||||
Delete the object and do a pack, it should get rid of the most current
|
||||
revision as well as the entire directory:
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> del root['blob']
|
||||
>>> transaction.commit()
|
||||
>>> packtime = new_time()
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, False, False]
|
||||
>>> os.path.exists(os.path.split(fns[0])[0])
|
||||
False
|
||||
|
||||
Clean up our blob directory and database:
|
||||
|
||||
>>> database.close()
|
||||
@@ -0,0 +1,53 @@
|
||||
=======================================
|
||||
Temporary directory handling with blobs
|
||||
=======================================
|
||||
|
||||
When creating uncommitted data files for a blob (e.g. by calling
|
||||
`blob.open('w')`) we need to decide where to create them. The decision depends
|
||||
on whether the blob is already stored in a database or not.
|
||||
|
||||
Case 1: Blobs that are not in a database yet
|
||||
============================================
|
||||
|
||||
Let's create a new blob and open it for writing::
|
||||
|
||||
>>> from ZODB.blob import Blob
|
||||
>>> b = Blob()
|
||||
>>> w = b.open('w')
|
||||
|
||||
The created file is in the default temporary directory::
|
||||
|
||||
>>> import tempfile
|
||||
>>> w.name.startswith(tempfile.gettempdir())
|
||||
True
|
||||
|
||||
>>> w.close()
|
||||
|
||||
Case 2: Blobs that are in a database
|
||||
====================================
|
||||
|
||||
For this case we instanciate a blob and add it to a database immediately.
|
||||
First, we need a datatabase with blob support::
|
||||
|
||||
>>> from ZODB.MappingStorage import MappingStorage
|
||||
>>> from ZODB.blob import BlobStorage
|
||||
>>> from ZODB.DB import DB
|
||||
>>> import os.path
|
||||
>>> base_storage = MappingStorage('test')
|
||||
>>> blob_dir = os.path.abspath('blobs')
|
||||
>>> blob_storage = BlobStorage(blob_dir, base_storage)
|
||||
>>> database = DB(blob_storage)
|
||||
|
||||
Now we create a blob and put it in the database. After that we open it for
|
||||
writing and expect the file to be in the blob temporary directory::
|
||||
|
||||
>>> blob = Blob()
|
||||
>>> connection = database.open()
|
||||
>>> connection.add(blob)
|
||||
>>> w = blob.open('w')
|
||||
>>> w.name.startswith(os.path.join(blob_dir, 'tmp'))
|
||||
True
|
||||
|
||||
>>> w.close()
|
||||
>>> database.close()
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
Transaction support for Blobs
|
||||
=============================
|
||||
|
||||
We need a database with a blob supporting storage::
|
||||
|
||||
>>> import ZODB.blob, transaction
|
||||
>>> blob_dir = 'blobs'
|
||||
>>> blob_storage = create_storage(blob_dir=blob_dir)
|
||||
>>> database = ZODB.DB(blob_storage)
|
||||
>>> connection1 = database.open()
|
||||
>>> root1 = connection1.root()
|
||||
|
||||
Putting a Blob into a Connection works like any other Persistent object::
|
||||
|
||||
>>> blob1 = ZODB.blob.Blob()
|
||||
>>> with blob1.open('w') as file:
|
||||
... _ = file.write(b'this is blob 1')
|
||||
>>> root1['blob1'] = blob1
|
||||
>>> 'blob1' in root1
|
||||
True
|
||||
|
||||
Aborting a blob add leaves the blob unchanged:
|
||||
|
||||
>>> transaction.abort()
|
||||
>>> 'blob1' in root1
|
||||
False
|
||||
|
||||
>>> blob1._p_oid
|
||||
>>> blob1._p_jar
|
||||
>>> with blob1.open() as fp:
|
||||
... fp.read()
|
||||
'this is blob 1'
|
||||
|
||||
It doesn't clear the file because there is no previously committed version:
|
||||
|
||||
>>> fname = blob1._p_blob_uncommitted
|
||||
>>> import os
|
||||
>>> os.path.exists(fname)
|
||||
True
|
||||
|
||||
Let's put the blob back into the root and commit the change:
|
||||
|
||||
>>> root1['blob1'] = blob1
|
||||
>>> transaction.commit()
|
||||
|
||||
Now, if we make a change and abort it, we'll return to the committed
|
||||
state:
|
||||
|
||||
>>> os.path.exists(fname)
|
||||
False
|
||||
>>> blob1._p_blob_uncommitted
|
||||
|
||||
>>> with blob1.open('w') as file:
|
||||
... _ = file.write(b'this is new blob 1')
|
||||
>>> with blob1.open() as fp:
|
||||
... fp.read()
|
||||
'this is new blob 1'
|
||||
>>> fname = blob1._p_blob_uncommitted
|
||||
>>> os.path.exists(fname)
|
||||
True
|
||||
|
||||
>>> transaction.abort()
|
||||
>>> os.path.exists(fname)
|
||||
False
|
||||
>>> blob1._p_blob_uncommitted
|
||||
|
||||
>>> with blob1.open() as fp:
|
||||
... fp.read()
|
||||
'this is blob 1'
|
||||
|
||||
Opening a blob gives us a filehandle. Getting data out of the
|
||||
resulting filehandle is accomplished via the filehandle's read method::
|
||||
|
||||
>>> connection2 = database.open()
|
||||
>>> root2 = connection2.root()
|
||||
>>> blob1a = root2['blob1']
|
||||
|
||||
>>> blob1afh1 = blob1a.open("r")
|
||||
>>> blob1afh1.read()
|
||||
'this is blob 1'
|
||||
|
||||
Let's make another filehandle for read only to blob1a. Each file
|
||||
handle has a reference to the (same) underlying blob::
|
||||
|
||||
>>> blob1afh2 = blob1a.open("r")
|
||||
>>> blob1afh2.blob is blob1afh1.blob
|
||||
True
|
||||
|
||||
Let's close the first filehandle we got from the blob::
|
||||
|
||||
>>> blob1afh1.close()
|
||||
|
||||
Let's abort this transaction, and ensure that the filehandles that we
|
||||
opened are still open::
|
||||
|
||||
>>> transaction.abort()
|
||||
>>> blob1afh2.read()
|
||||
'this is blob 1'
|
||||
|
||||
>>> blob1afh2.close()
|
||||
|
||||
If we open a blob for append, writing any number of bytes to the
|
||||
blobfile should result in the blob being marked "dirty" in the
|
||||
connection (we just aborted above, so the object should be "clean"
|
||||
when we start)::
|
||||
|
||||
>>> bool(blob1a._p_changed)
|
||||
False
|
||||
>>> with blob1a.open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1'
|
||||
>>> with blob1a.open('a') as blob1afh3:
|
||||
... assert(bool(blob1a._p_changed))
|
||||
... _ = blob1afh3.write(b'woot!')
|
||||
>>> blob1afh3.close()
|
||||
|
||||
We can open more than one blob object during the course of a single
|
||||
transaction::
|
||||
|
||||
>>> blob2 = ZODB.blob.Blob()
|
||||
>>> with blob2.open('w') as file:
|
||||
... _ = file.write(b'this is blob 3')
|
||||
>>> root2['blob2'] = blob2
|
||||
>>> transaction.commit()
|
||||
|
||||
Since we committed the current transaction above, the aggregate
|
||||
changes we've made to blob, blob1a (these refer to the same object) and
|
||||
blob2 (a different object) should be evident::
|
||||
|
||||
>>> with blob1.open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1woot!'
|
||||
>>> with blob1a.open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1woot!'
|
||||
>>> with blob2.open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 3'
|
||||
|
||||
We shouldn't be able to persist a blob filehandle at commit time
|
||||
(although the exception which is raised when an object cannot be
|
||||
pickled appears to be particulary unhelpful for casual users at the
|
||||
moment)::
|
||||
|
||||
>>> with blob1.open('r') as f:
|
||||
... root1['wontwork'] = f
|
||||
... transaction.commit()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
TypeError: ...
|
||||
|
||||
Abort for good measure::
|
||||
|
||||
>>> transaction.abort()
|
||||
|
||||
Attempting to change a blob simultaneously from two different
|
||||
connections should result in a write conflict error::
|
||||
|
||||
>>> tm1 = transaction.TransactionManager()
|
||||
>>> tm2 = transaction.TransactionManager()
|
||||
>>> root3 = database.open(transaction_manager=tm1).root()
|
||||
>>> root4 = database.open(transaction_manager=tm2).root()
|
||||
>>> blob1c3 = root3['blob1']
|
||||
>>> blob1c4 = root4['blob1']
|
||||
>>> with blob1c3.open('a') as blob1c3fh1:
|
||||
... _ = blob1c3fh1.write(b'this is from connection 3')
|
||||
>>> with blob1c4.open('a') as blob1c4fh1:
|
||||
... _ = blob1c4fh1.write(b'this is from connection 4')
|
||||
>>> tm1.commit()
|
||||
>>> with root3['blob1'].open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1woot!this is from connection 3'
|
||||
>>> tm2.commit()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ConflictError: database conflict error (oid 0x01, class ZODB.blob.Blob...)
|
||||
|
||||
After the conflict, the winning transaction's result is visible on both
|
||||
connections::
|
||||
|
||||
>>> with root3['blob1'].open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1woot!this is from connection 3'
|
||||
>>> tm2.abort()
|
||||
>>> with root4['blob1'].open('r') as fp:
|
||||
... fp.read()
|
||||
'this is blob 1woot!this is from connection 3'
|
||||
|
||||
You can't commit a transaction while blob files are open:
|
||||
|
||||
>>> f = root3['blob1'].open('w')
|
||||
>>> tm1.commit()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: Can't commit with opened blobs.
|
||||
|
||||
>>> f.close()
|
||||
>>> tm1.abort()
|
||||
>>> f = root3['blob1'].open('w')
|
||||
>>> f.close()
|
||||
|
||||
>>> f = root3['blob1'].open('r')
|
||||
>>> tm1.commit()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: Can't commit with opened blobs.
|
||||
>>> f.close()
|
||||
>>> tm1.abort()
|
||||
|
||||
Savepoints and Blobs
|
||||
--------------------
|
||||
|
||||
We do support optimistic savepoints:
|
||||
|
||||
>>> connection5 = database.open()
|
||||
>>> root5 = connection5.root()
|
||||
>>> blob = ZODB.blob.Blob()
|
||||
>>> with blob.open("w") as blob_fh:
|
||||
... _ = blob_fh.write(b"I'm a happy blob.")
|
||||
>>> root5['blob'] = blob
|
||||
>>> transaction.commit()
|
||||
>>> with root5['blob'].open("r") as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob."
|
||||
>>> with root5['blob'].open("a") as blob_fh:
|
||||
... _ = blob_fh.write(b" And I'm singing.")
|
||||
>>> with root5['blob'].open("r") as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob. And I'm singing."
|
||||
>>> savepoint = transaction.savepoint(optimistic=True)
|
||||
|
||||
>>> with root5['blob'].open("r") as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob. And I'm singing."
|
||||
|
||||
Savepoints store the blobs in temporary directories in the temporary
|
||||
directory of the blob storage:
|
||||
|
||||
>>> len([name for name in os.listdir(os.path.join(blob_dir, 'tmp'))
|
||||
... if name.startswith('savepoint')])
|
||||
1
|
||||
|
||||
After committing the transaction, the temporary savepoint files are moved to
|
||||
the committed location again:
|
||||
|
||||
>>> transaction.commit()
|
||||
>>> len([name for name in os.listdir(os.path.join(blob_dir, 'tmp'))
|
||||
... if name.startswith('savepoint')])
|
||||
0
|
||||
|
||||
We support non-optimistic savepoints too:
|
||||
|
||||
>>> with root5['blob'].open("a") as file:
|
||||
... _ = file.write(b" And I'm dancing.")
|
||||
>>> with root5['blob'].open("r") as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob. And I'm singing. And I'm dancing."
|
||||
>>> savepoint = transaction.savepoint()
|
||||
|
||||
Again, the savepoint creates a new savepoints directory:
|
||||
|
||||
>>> len([name for name in os.listdir(os.path.join(blob_dir, 'tmp'))
|
||||
... if name.startswith('savepoint')])
|
||||
1
|
||||
|
||||
>>> with root5['blob'].open("w") as file:
|
||||
... _ = file.write(b" And the weather is beautiful.")
|
||||
>>> savepoint.rollback()
|
||||
|
||||
>>> with root5['blob'].open("r") as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob. And I'm singing. And I'm dancing."
|
||||
>>> transaction.abort()
|
||||
|
||||
The savepoint blob directory gets cleaned up on an abort:
|
||||
|
||||
>>> len([name for name in os.listdir(os.path.join(blob_dir, 'tmp'))
|
||||
... if name.startswith('savepoint')])
|
||||
0
|
||||
|
||||
Reading Blobs outside of a transaction
|
||||
--------------------------------------
|
||||
|
||||
If you want to read from a Blob outside of transaction boundaries (e.g. to
|
||||
stream a file to the browser), committed method to get the name of a
|
||||
file that can be opened.
|
||||
|
||||
>>> connection6 = database.open()
|
||||
>>> root6 = connection6.root()
|
||||
>>> blob = ZODB.blob.Blob()
|
||||
>>> with blob.open("w") as blob_fh:
|
||||
... _ = blob_fh.write(b"I'm a happy blob.")
|
||||
>>> root6['blob'] = blob
|
||||
>>> transaction.commit()
|
||||
>>> with open(blob.committed()) as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob."
|
||||
|
||||
We can also read committed data by calling open with a 'c' flag:
|
||||
|
||||
>>> f = blob.open('c')
|
||||
|
||||
This just returns a regular file object:
|
||||
|
||||
>>> type(f) == file_type
|
||||
True
|
||||
|
||||
and doesn't prevent us from opening the blob for writing:
|
||||
|
||||
>>> with blob.open('w') as file:
|
||||
... _ = file.write(b'x')
|
||||
>>> with blob.open() as fp: fp.read()
|
||||
'x'
|
||||
|
||||
>>> f.read()
|
||||
"I'm a happy blob."
|
||||
|
||||
>>> f.close()
|
||||
>>> transaction.abort()
|
||||
|
||||
An exception is raised if we call committed on a blob that has
|
||||
uncommitted changes:
|
||||
|
||||
>>> blob = ZODB.blob.Blob()
|
||||
>>> blob.committed()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> blob.open('c')
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> with blob.open('w') as file:
|
||||
... _ = file.write(b"I'm a happy blob.")
|
||||
>>> root6['blob6'] = blob
|
||||
>>> blob.committed()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> blob.open('c')
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> s = transaction.savepoint()
|
||||
>>> blob.committed()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> blob.open('c')
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
BlobError: Uncommitted changes
|
||||
|
||||
>>> transaction.commit()
|
||||
>>> with open(blob.committed()) as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob."
|
||||
|
||||
You can't open a committed blob file for writing:
|
||||
|
||||
>>> try:
|
||||
... open(blob.committed(), 'w') # doctest: +ELLIPSIS
|
||||
... except:
|
||||
... # Produces IOError in Py2 and PermissionError in Py3
|
||||
... print('Error raised.')
|
||||
Error raised.
|
||||
|
||||
tpc_abort
|
||||
---------
|
||||
|
||||
If a transaction is aborted in the middle of 2-phase commit, any data
|
||||
stored are discarded.
|
||||
|
||||
>>> olddata, oldserial = blob_storage.load(blob._p_oid, '')
|
||||
>>> from ZODB.Connection import TransactionMetaData
|
||||
>>> t = TransactionMetaData()
|
||||
>>> blob_storage.tpc_begin(t)
|
||||
>>> with open('blobfile', 'wb') as file:
|
||||
... _ = file.write(b'This data should go away')
|
||||
>>> blob_storage.storeBlob(blob._p_oid, oldserial, olddata, 'blobfile',
|
||||
... '', t)
|
||||
>>> new_oid = blob_storage.new_oid()
|
||||
>>> with open('blobfile2', 'wb') as file:
|
||||
... _ = file.write(b'This data should go away too')
|
||||
>>> blob_storage.storeBlob(new_oid, '\0'*8, olddata, 'blobfile2',
|
||||
... '', t)
|
||||
>>> bool(blob_storage.tpc_vote(t))
|
||||
False
|
||||
>>> blob_storage.tpc_abort(t)
|
||||
|
||||
Now, the serial for the existing blob should be the same:
|
||||
|
||||
>>> blob_storage.load(blob._p_oid, '') == (olddata, oldserial)
|
||||
True
|
||||
|
||||
The old data should be unaffected:
|
||||
|
||||
>>> with open(blob_storage.loadBlob(blob._p_oid, oldserial)) as fp:
|
||||
... fp.read()
|
||||
"I'm a happy blob."
|
||||
|
||||
Similarly, the new object wasn't added to the storage:
|
||||
|
||||
>>> blob_storage.load(new_oid, '')
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
POSKeyError: 0x...
|
||||
|
||||
.. clean up
|
||||
|
||||
>>> tm1.abort()
|
||||
>>> tm2.abort()
|
||||
>>> database.close()
|
||||
@@ -0,0 +1,164 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2005 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
Packing support for blob data
|
||||
=============================
|
||||
|
||||
Set up:
|
||||
|
||||
>>> from ZODB.MappingStorage import MappingStorage
|
||||
>>> from ZODB.serialize import referencesf
|
||||
>>> from ZODB.blob import Blob, BlobStorage
|
||||
>>> from ZODB import utils
|
||||
>>> from ZODB.DB import DB
|
||||
>>> import transaction
|
||||
>>> storagefile = 'Data.fs'
|
||||
>>> blob_dir = 'blobs'
|
||||
|
||||
A helper method to assure a unique timestamp across multiple platforms:
|
||||
|
||||
>>> from ZODB.tests.testblob import new_time
|
||||
|
||||
UNDOING
|
||||
=======
|
||||
|
||||
See blob_packing.txt.
|
||||
|
||||
NON-UNDOING
|
||||
===========
|
||||
|
||||
We need an database with a NON-undoing blob supporting storage:
|
||||
|
||||
>>> base_storage = MappingStorage('storage')
|
||||
>>> blob_storage = BlobStorage(blob_dir, base_storage)
|
||||
>>> database = DB(blob_storage)
|
||||
|
||||
Create our root object:
|
||||
|
||||
>>> connection1 = database.open()
|
||||
>>> root = connection1.root()
|
||||
|
||||
Put some revisions of a blob object in our database and on the filesystem:
|
||||
|
||||
>>> import time, os
|
||||
>>> tids = []
|
||||
>>> times = []
|
||||
>>> nothing = transaction.begin()
|
||||
>>> times.append(new_time())
|
||||
>>> blob = Blob()
|
||||
>>> with blob.open('w') as file:
|
||||
... _ = file.write(b'this is blob data 0')
|
||||
>>> root['blob'] = blob
|
||||
>>> transaction.commit()
|
||||
>>> tids.append(blob_storage._tid)
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> times.append(new_time())
|
||||
>>> with root['blob'].open('w') as file:
|
||||
... _ = file.write(b'this is blob data 1')
|
||||
>>> transaction.commit()
|
||||
>>> tids.append(blob_storage._tid)
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> times.append(new_time())
|
||||
>>> with root['blob'].open('w') as file:
|
||||
... _ = file.write(b'this is blob data 2')
|
||||
>>> transaction.commit()
|
||||
>>> tids.append(blob_storage._tid)
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> times.append(new_time())
|
||||
>>> with root['blob'].open('w') as file:
|
||||
... _ = file.write(b'this is blob data 3')
|
||||
>>> transaction.commit()
|
||||
>>> tids.append(blob_storage._tid)
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> times.append(new_time())
|
||||
>>> with root['blob'].open('w') as file:
|
||||
... _ = file.write(b'this is blob data 4')
|
||||
>>> transaction.commit()
|
||||
>>> tids.append(blob_storage._tid)
|
||||
|
||||
>>> oid = root['blob']._p_oid
|
||||
>>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[True, True, True, True, True]
|
||||
|
||||
Get our blob filenames for this oid.
|
||||
|
||||
>>> fns = [ blob_storage.fshelper.getBlobFilename(oid, x) for x in tids ]
|
||||
|
||||
Do a pack to the slightly before the first revision was written:
|
||||
|
||||
>>> packtime = times[0]
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, False, True]
|
||||
|
||||
Do a pack to now:
|
||||
|
||||
>>> packtime = new_time()
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, False, True]
|
||||
|
||||
Delete the object and do a pack, it should get rid of the most current
|
||||
revision as well as the entire directory:
|
||||
|
||||
>>> nothing = transaction.begin()
|
||||
>>> del root['blob']
|
||||
>>> transaction.commit()
|
||||
>>> packtime = new_time()
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
>>> [ os.path.exists(x) for x in fns ]
|
||||
[False, False, False, False, False]
|
||||
>>> os.path.exists(os.path.split(fns[0])[0])
|
||||
False
|
||||
|
||||
Avoiding parallel packs
|
||||
=======================
|
||||
|
||||
Blob packing (similar to FileStorage) can only be run once at a time. For
|
||||
this, a flag (_blobs_pack_is_in_progress) is set. If the pack method is called
|
||||
while this flag is set, it will refuse to perform another pack, until the flag
|
||||
is reset:
|
||||
|
||||
>>> blob_storage._blobs_pack_is_in_progress
|
||||
False
|
||||
>>> blob_storage._blobs_pack_is_in_progress = True
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
Traceback (most recent call last):
|
||||
BlobStorageError: Already packing
|
||||
>>> blob_storage._blobs_pack_is_in_progress = False
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
|
||||
We can also see, that the flag is set during the pack, by leveraging the
|
||||
knowledge that the underlying storage's pack method is also called:
|
||||
|
||||
>>> import six
|
||||
>>> def dummy_pack(time, ref):
|
||||
... six.print_(
|
||||
... "_blobs_pack_is_in_progress =",
|
||||
... blob_storage._blobs_pack_is_in_progress)
|
||||
... return base_pack(time, ref)
|
||||
>>> base_pack = base_storage.pack
|
||||
>>> base_storage.pack = dummy_pack
|
||||
>>> blob_storage.pack(packtime, referencesf)
|
||||
_blobs_pack_is_in_progress = True
|
||||
>>> blob_storage._blobs_pack_is_in_progress
|
||||
False
|
||||
>>> base_storage.pack = base_pack
|
||||
|
||||
>>> database.close()
|
||||
@@ -0,0 +1,16 @@
|
||||
<component>
|
||||
<sectiontype
|
||||
name="hexstorage"
|
||||
datatype="ZODB.tests.hexstorage.ZConfigHex"
|
||||
implements="ZODB.storage"
|
||||
>
|
||||
<section type="ZODB.storage" name="*" attribute="base" required="yes" />
|
||||
</sectiontype>
|
||||
<sectiontype
|
||||
name="serverhexstorage"
|
||||
datatype="ZODB.tests.hexstorage.ZConfigServerHex"
|
||||
implements="ZODB.storage"
|
||||
>
|
||||
<section type="ZODB.storage" name="*" attribute="base" required="yes" />
|
||||
</sectiontype>
|
||||
</component>
|
||||
@@ -0,0 +1,66 @@
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright (c) 2002 Zope Foundation and Contributors.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# This software is subject to the provisions of the Zope Public License,
|
||||
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
|
||||
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
|
||||
# FOR A PARTICULAR PURPOSE
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
"""Functional test to produce a dangling reference."""
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
|
||||
import transaction
|
||||
from ZODB.FileStorage import FileStorage
|
||||
from ZODB import DB
|
||||
|
||||
from persistent import Persistent
|
||||
|
||||
class P(Persistent):
|
||||
pass
|
||||
|
||||
def create_dangling_ref(db):
|
||||
rt = db.open().root()
|
||||
|
||||
rt[1] = o1 = P()
|
||||
transaction.get().note(u"create o1")
|
||||
transaction.commit()
|
||||
|
||||
rt[2] = o2 = P()
|
||||
transaction.get().note(u"create o2")
|
||||
transaction.commit()
|
||||
|
||||
c = o1.child = P()
|
||||
transaction.get().note(u"set child on o1")
|
||||
transaction.commit()
|
||||
|
||||
o1.child = P()
|
||||
transaction.get().note(u"replace child on o1")
|
||||
transaction.commit()
|
||||
|
||||
time.sleep(2)
|
||||
# The pack should remove the reference to c, because it is no
|
||||
# longer referenced from o1. But the object still exists and has
|
||||
# an oid, so a new commit of it won't create a new object.
|
||||
db.pack()
|
||||
|
||||
print(repr(c._p_oid))
|
||||
o2.child = c
|
||||
transaction.get().note(u"set child on o2")
|
||||
transaction.commit()
|
||||
|
||||
def main():
|
||||
fs = FileStorage(u"dangle.fs")
|
||||
db = DB(fs)
|
||||
create_dangling_ref(db)
|
||||
db.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,377 @@
|
||||
=====================
|
||||
Connection Management
|
||||
=====================
|
||||
|
||||
|
||||
Here we exercise the connection management done by the DB class.
|
||||
|
||||
>>> from ZODB import DB
|
||||
>>> from ZODB.MappingStorage import MappingStorage as Storage
|
||||
|
||||
Capturing log messages from DB is important for some of the examples:
|
||||
|
||||
>>> from zope.testing.loggingsupport import InstalledHandler
|
||||
>>> handler = InstalledHandler('ZODB.DB')
|
||||
|
||||
Create a storage, and wrap it in a DB wrapper:
|
||||
|
||||
>>> st = Storage()
|
||||
>>> db = DB(st)
|
||||
|
||||
By default, we can open 7 connections without any log messages:
|
||||
|
||||
>>> conns = [db.open() for dummy in range(7)]
|
||||
>>> handler.records
|
||||
[]
|
||||
|
||||
Open one more, and we get a warning:
|
||||
|
||||
>>> conns.append(db.open())
|
||||
>>> len(handler.records)
|
||||
1
|
||||
>>> msg = handler.records[0]
|
||||
>>> import six
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB WARNING DB.open() has 8 open connections with a pool_size of 7
|
||||
|
||||
Open 6 more, and we get 6 more warnings:
|
||||
|
||||
>>> conns.extend([db.open() for dummy in range(6)])
|
||||
>>> len(conns)
|
||||
14
|
||||
>>> len(handler.records)
|
||||
7
|
||||
>>> msg = handler.records[-1]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB WARNING DB.open() has 14 open connections with a pool_size of 7
|
||||
|
||||
Add another, so that it's more than twice the default, and the level
|
||||
rises to critical:
|
||||
|
||||
>>> conns.append(db.open())
|
||||
>>> len(conns)
|
||||
15
|
||||
>>> len(handler.records)
|
||||
8
|
||||
>>> msg = handler.records[-1]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB CRITICAL DB.open() has 15 open connections with a pool_size of 7
|
||||
|
||||
While it's boring, it's important to verify that the same relationships
|
||||
hold if the default pool size is overridden.
|
||||
|
||||
>>> handler.clear()
|
||||
>>> db.close()
|
||||
>>> st = Storage()
|
||||
>>> PS = 2 # smaller pool size
|
||||
>>> db = DB(st, pool_size=PS)
|
||||
>>> conns = [db.open() for dummy in range(PS)]
|
||||
>>> handler.records
|
||||
[]
|
||||
|
||||
A warning for opening one more:
|
||||
|
||||
>>> conns.append(db.open())
|
||||
>>> len(handler.records)
|
||||
1
|
||||
>>> msg = handler.records[0]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB WARNING DB.open() has 3 open connections with a pool_size of 2
|
||||
|
||||
More warnings through 4 connections:
|
||||
|
||||
>>> conns.extend([db.open() for dummy in range(PS-1)])
|
||||
>>> len(conns)
|
||||
4
|
||||
>>> len(handler.records)
|
||||
2
|
||||
>>> msg = handler.records[-1]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB WARNING DB.open() has 4 open connections with a pool_size of 2
|
||||
|
||||
And critical for going beyond that:
|
||||
|
||||
>>> conns.append(db.open())
|
||||
>>> len(conns)
|
||||
5
|
||||
>>> len(handler.records)
|
||||
3
|
||||
>>> msg = handler.records[-1]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB CRITICAL DB.open() has 5 open connections with a pool_size of 2
|
||||
|
||||
We can change the pool size on the fly:
|
||||
|
||||
>>> handler.clear()
|
||||
>>> db.setPoolSize(6)
|
||||
>>> conns.append(db.open())
|
||||
>>> handler.records # no log msg -- the pool is bigger now
|
||||
[]
|
||||
>>> conns.append(db.open()) # but one more and there's a warning again
|
||||
>>> len(handler.records)
|
||||
1
|
||||
>>> msg = handler.records[0]
|
||||
>>> six.print_(msg.name, msg.levelname, msg.getMessage())
|
||||
ZODB.DB WARNING DB.open() has 7 open connections with a pool_size of 6
|
||||
|
||||
Enough of that.
|
||||
|
||||
>>> handler.clear()
|
||||
>>> db.close()
|
||||
|
||||
More interesting is the stack-like nature of connection reuse. So long as
|
||||
we keep opening new connections, and keep them alive, all connections
|
||||
returned are distinct:
|
||||
|
||||
>>> st = Storage()
|
||||
>>> db = DB(st)
|
||||
>>> c1 = db.open()
|
||||
>>> c1.cacheMinimize()
|
||||
>>> c2 = db.open()
|
||||
>>> c3 = db.open()
|
||||
>>> c1 is c2 or c1 is c3 or c2 is c3
|
||||
False
|
||||
|
||||
Let's put some markers on the connections, so we can identify these
|
||||
specific objects later:
|
||||
|
||||
>>> c1.MARKER = 'c1'
|
||||
>>> c2.MARKER = 'c2'
|
||||
>>> c3.MARKER = 'c3'
|
||||
|
||||
Now explicitly close c1 and c2:
|
||||
|
||||
>>> c1.close()
|
||||
>>> c2.close()
|
||||
|
||||
Reaching into the internals, we can see that db's connection pool now has
|
||||
two connections available for reuse, and knows about three connections in
|
||||
all:
|
||||
|
||||
>>> pool = db.pool
|
||||
>>> len(pool.available)
|
||||
2
|
||||
>>> len(pool.all)
|
||||
3
|
||||
|
||||
Since we closed c2 last, it's at the top of the available stack, so will
|
||||
be reused by the next open():
|
||||
|
||||
>>> c1 = db.open()
|
||||
>>> c1.MARKER
|
||||
'c2'
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(1, 3)
|
||||
|
||||
>>> c3.close() # now the stack has c3 on top, then c1
|
||||
>>> c2 = db.open()
|
||||
>>> c2.MARKER
|
||||
'c3'
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(1, 3)
|
||||
>>> c3 = db.open()
|
||||
>>> c3.MARKER
|
||||
'c1'
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(0, 3)
|
||||
|
||||
It's a bit more complicated though. The connection pool tries to keep
|
||||
connections with larger caches at the top of the stack. It does this
|
||||
by having connections with smaller caches "sink" below connections with
|
||||
larger caches when they are closed.
|
||||
|
||||
To see this, we'll add some objects to the caches:
|
||||
|
||||
>>> for i in range(10):
|
||||
... c1.root()[i] = c1.root().__class__()
|
||||
>>> import transaction
|
||||
>>> transaction.commit()
|
||||
>>> c1._cache.cache_non_ghost_count
|
||||
11
|
||||
|
||||
>>> for i in range(5):
|
||||
... _ = len(c2.root()[i])
|
||||
>>> c2._cache.cache_non_ghost_count
|
||||
6
|
||||
|
||||
Now, we'll close the connections and get them back:
|
||||
|
||||
>>> c1.close()
|
||||
>>> c2.close()
|
||||
>>> c3.close()
|
||||
|
||||
We closed c3 last, but c1 is the biggest, so we get c1 on the next
|
||||
open:
|
||||
|
||||
>>> db.open() is c1
|
||||
True
|
||||
|
||||
Similarly, c2 is the next buggest, so we get that next:
|
||||
|
||||
>>> db.open() is c2
|
||||
True
|
||||
|
||||
and finally c3:
|
||||
|
||||
>>> db.open() is c3
|
||||
True
|
||||
|
||||
What about the 3 in pool.all? We've seen that closing connections doesn't
|
||||
reduce pool.all, and it would be bad if DB kept connections alive forever.
|
||||
|
||||
In fact pool.all is a "weak set" of connections -- it holds weak references
|
||||
to connections. That alone doesn't keep connection objects alive. The
|
||||
weak set allows DB's statistics methods to return info about connections
|
||||
that are still alive.
|
||||
|
||||
|
||||
>>> len(db.cacheDetailSize()) # one result for each connection's cache
|
||||
3
|
||||
|
||||
If a connection object is abandoned (it becomes unreachable), then it
|
||||
will vanish from pool.all automatically. However, connections are
|
||||
involved in cycles, so exactly when a connection vanishes from
|
||||
pool.all isn't predictable. It can be forced (on most platforms but
|
||||
not Jython) by running gc.collect():
|
||||
|
||||
>>> import gc, sys
|
||||
>>> dummy = gc.collect()
|
||||
>>> len(pool.all)
|
||||
3
|
||||
>>> c3 = None
|
||||
>>> dummy = gc.collect() # removes c3 from pool.all
|
||||
>>> len(pool.all) if not sys.platform.startswith("java") else 2
|
||||
2
|
||||
|
||||
Note that c3 is really gone; in particular it didn't get added back to
|
||||
the stack of available connections by magic:
|
||||
|
||||
>>> len(pool.available)
|
||||
0
|
||||
|
||||
Nothing in that last block should have logged any msgs:
|
||||
|
||||
>>> handler.records
|
||||
[]
|
||||
|
||||
If "too many" connections are open, then closing one may kick an older
|
||||
closed one out of the available connection stack.
|
||||
|
||||
>>> db.close()
|
||||
>>> st = Storage()
|
||||
>>> db = DB(st, pool_size=3)
|
||||
>>> conns = [db.open() for dummy in range(6)]
|
||||
>>> conns[0].cacheMinimize()
|
||||
>>> len(handler.records) # 3 warnings for the "excess" connections
|
||||
3
|
||||
>>> pool = db.pool
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(0, 6)
|
||||
|
||||
Let's mark them:
|
||||
|
||||
>>> for i, c in enumerate(conns):
|
||||
... c.MARKER = i
|
||||
|
||||
Closing connections adds them to the stack:
|
||||
|
||||
>>> for i in range(3):
|
||||
... conns[i].close()
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(3, 6)
|
||||
>>> del conns[:3] # leave the ones with MARKERs 3, 4 and 5
|
||||
|
||||
Closing another one will purge the one with MARKER 0 from the stack
|
||||
(since it was the first added to the stack):
|
||||
|
||||
>>> [c.MARKER for (t, c) in pool.available]
|
||||
[0, 1, 2]
|
||||
>>> conns[0].close() # MARKER 3
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(3, 5)
|
||||
>>> [c.MARKER for (t, c) in pool.available]
|
||||
[1, 2, 3]
|
||||
|
||||
Similarly for the other two:
|
||||
|
||||
>>> conns[1].close(); conns[2].close()
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(3, 3)
|
||||
>>> [c.MARKER for (t, c) in pool.available]
|
||||
[3, 4, 5]
|
||||
|
||||
Reducing the pool size may also purge the oldest closed connections:
|
||||
|
||||
>>> db.setPoolSize(2) # gets rid of MARKER 3
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(2, 2)
|
||||
>>> [c.MARKER for (t, c) in pool.available]
|
||||
[4, 5]
|
||||
|
||||
Since MARKER 5 is still the last one added to the stack, it will be the
|
||||
first popped:
|
||||
|
||||
>>> c1 = db.open(); c2 = db.open()
|
||||
>>> c1.MARKER, c2.MARKER
|
||||
(5, 4)
|
||||
>>> len(pool.available), len(pool.all)
|
||||
(0, 2)
|
||||
|
||||
Next: when a closed Connection is removed from .available due to exceeding
|
||||
pool_size, that Connection's cache is cleared (this behavior was new in
|
||||
ZODB 3.6b6). While user code may still hold a reference to that
|
||||
Connection, once it vanishes from .available it's really not usable for
|
||||
anything sensible (it can never be in the open state again). Waiting for
|
||||
gc to reclaim the Connection and its cache eventually works, but that can
|
||||
take "a long time" and caches can hold on to many objects, and limited
|
||||
resources (like RDB connections), for the duration.
|
||||
|
||||
>>> db.close()
|
||||
>>> st = Storage()
|
||||
>>> db = DB(st, pool_size=2)
|
||||
>>> conn0 = db.open()
|
||||
>>> conn0.cacheMinimize(); import gc; _ = gc.collect() # See fix84.rst
|
||||
>>> len(conn0._cache) # empty now
|
||||
0
|
||||
>>> import transaction
|
||||
>>> conn0.root()['a'] = 1
|
||||
>>> transaction.commit()
|
||||
>>> len(conn0._cache) # but now the cache holds the root object
|
||||
1
|
||||
|
||||
Now open more connections so that the total exceeds pool_size (2):
|
||||
|
||||
>>> conn1 = db.open(); _ = conn1.root()['a']
|
||||
>>> conn2 = db.open(); _ = conn2.root()['a']
|
||||
|
||||
Note that we accessed the objects in the new connections so they would
|
||||
be of the same size, so that when they get closed, they don't sink
|
||||
below conn0.
|
||||
|
||||
>>> pool = db.pool
|
||||
>>> len(pool.all), len(pool.available) # all Connections are in use
|
||||
(3, 0)
|
||||
|
||||
Return pool_size (2) Connections to the pool:
|
||||
|
||||
>>> conn0.close()
|
||||
>>> conn1.close()
|
||||
>>> len(pool.all), len(pool.available)
|
||||
(3, 2)
|
||||
>>> len(conn0._cache) # nothing relevant has changed yet
|
||||
1
|
||||
|
||||
When we close the third connection, conn0 will be booted from .all, and
|
||||
we expect its cache to be cleared then:
|
||||
|
||||
>>> conn2.close()
|
||||
>>> len(pool.all), len(pool.available)
|
||||
(2, 2)
|
||||
>>> len(conn0._cache) # conn0's cache is empty again
|
||||
0
|
||||
>>> del conn0, conn1, conn2
|
||||
|
||||
Clean up.
|
||||
|
||||
>>> st.close()
|
||||
>>> handler.uninstall()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user