|
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091 |
- # -*- test-case-name: twisted.spread.test.test_jelly -*-
- # Copyright (c) Twisted Matrix Laboratories.
- # See LICENSE for details.
-
- """
- S-expression-based persistence of python objects.
-
- It does something very much like L{Pickle<pickle>}; however, pickle's main goal
- seems to be efficiency (both in space and time); jelly's main goals are
- security, human readability, and portability to other environments.
-
- This is how Jelly converts various objects to s-expressions.
-
- Boolean::
- True --> ['boolean', 'true']
-
- Integer::
- 1 --> 1
-
- List::
- [1, 2] --> ['list', 1, 2]
-
- String::
- \"hello\" --> \"hello\"
-
- Float::
- 2.3 --> 2.3
-
- Dictionary::
- {'a': 1, 'b': 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]]
-
- Module::
- UserString --> ['module', 'UserString']
-
- Class::
- UserString.UserString --> ['class', ['module', 'UserString'], 'UserString']
-
- Function::
- string.join --> ['function', 'join', ['module', 'string']]
-
- Instance: s is an instance of UserString.UserString, with a __dict__
- {'data': 'hello'}::
- [\"UserString.UserString\", ['dictionary', ['data', 'hello']]]
-
- Class Method: UserString.UserString.center::
- ['method', 'center', ['None'], ['class', ['module', 'UserString'],
- 'UserString']]
-
- Instance Method: s.center, where s is an instance of UserString.UserString::
- ['method', 'center', ['instance', ['reference', 1, ['class',
- ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]],
- ['dereference', 1]]
-
- The Python 2.x C{sets.Set} and C{sets.ImmutableSet} classes are
- serialized to the same thing as the builtin C{set} and C{frozenset}
- classes. (This is only relevant if you are communicating with a
- version of jelly running on an older version of Python.)
-
- @author: Glyph Lefkowitz
-
- """
-
- import copy
- import datetime
- import decimal
-
- # System Imports
- import types
- import warnings
- from functools import reduce
-
- from zope.interface import implementer
-
- from incremental import Version
-
- from twisted.persisted.crefutil import (
- NotKnown,
- _Container,
- _Dereference,
- _DictKeyAndValue,
- _InstanceMethod,
- _Tuple,
- )
-
- # Twisted Imports
- from twisted.python.compat import nativeString
- from twisted.python.deprecate import deprecatedModuleAttribute
- from twisted.python.reflect import namedAny, namedObject, qual
- from twisted.spread.interfaces import IJellyable, IUnjellyable
-
- DictTypes = (dict,)
-
- None_atom = b"None" # N
- # code
- class_atom = b"class" # c
- module_atom = b"module" # m
- function_atom = b"function" # f
-
- # references
- dereference_atom = b"dereference" # D
- persistent_atom = b"persistent" # p
- reference_atom = b"reference" # r
-
- # mutable collections
- dictionary_atom = b"dictionary" # d
- list_atom = b"list" # l
- set_atom = b"set"
-
- # immutable collections
- # (assignment to __dict__ and __class__ still might go away!)
- tuple_atom = b"tuple" # t
- instance_atom = b"instance" # i
- frozenset_atom = b"frozenset"
-
-
- deprecatedModuleAttribute(
- Version("Twisted", 15, 0, 0),
- "instance_atom is unused within Twisted.",
- "twisted.spread.jelly",
- "instance_atom",
- )
-
- # errors
- unpersistable_atom = b"unpersistable" # u
- unjellyableRegistry = {}
- unjellyableFactoryRegistry = {}
-
-
- def _createBlank(cls):
- """
- Given an object, if that object is a type, return a new, blank instance
- of that type which has not had C{__init__} called on it. If the object
- is not a type, return L{None}.
-
- @param cls: The type (or class) to create an instance of.
- @type cls: L{type} or something else that cannot be
- instantiated.
-
- @return: a new blank instance or L{None} if C{cls} is not a class or type.
- """
- if isinstance(cls, type):
- return cls.__new__(cls)
-
-
- def _newInstance(cls, state):
- """
- Make a new instance of a class without calling its __init__ method.
-
- @param state: A C{dict} used to update C{inst.__dict__} either directly or
- via C{__setstate__}, if available.
-
- @return: A new instance of C{cls}.
- """
- instance = _createBlank(cls)
-
- def defaultSetter(state):
- instance.__dict__ = state
-
- setter = getattr(instance, "__setstate__", defaultSetter)
- setter(state)
- return instance
-
-
- def _maybeClass(classnamep):
- isObject = isinstance(classnamep, type)
-
- if isObject:
- classnamep = qual(classnamep)
-
- if not isinstance(classnamep, bytes):
- classnamep = classnamep.encode("utf-8")
-
- return classnamep
-
-
- def setUnjellyableForClass(classname, unjellyable):
- """
- Set which local class will represent a remote type.
-
- If you have written a Copyable class that you expect your client to be
- receiving, write a local "copy" class to represent it, then call::
-
- jellier.setUnjellyableForClass('module.package.Class', MyCopier).
-
- Call this at the module level immediately after its class
- definition. MyCopier should be a subclass of RemoteCopy.
-
- The classname may be a special tag returned by
- 'Copyable.getTypeToCopyFor' rather than an actual classname.
-
- This call is also for cached classes, since there will be no
- overlap. The rules are the same.
- """
-
- global unjellyableRegistry
- classname = _maybeClass(classname)
- unjellyableRegistry[classname] = unjellyable
- globalSecurity.allowTypes(classname)
-
-
- def setUnjellyableFactoryForClass(classname, copyFactory):
- """
- Set the factory to construct a remote instance of a type::
-
- jellier.setUnjellyableFactoryForClass('module.package.Class', MyFactory)
-
- Call this at the module level immediately after its class definition.
- C{copyFactory} should return an instance or subclass of
- L{RemoteCopy<pb.RemoteCopy>}.
-
- Similar to L{setUnjellyableForClass} except it uses a factory instead
- of creating an instance.
- """
-
- global unjellyableFactoryRegistry
- classname = _maybeClass(classname)
- unjellyableFactoryRegistry[classname] = copyFactory
- globalSecurity.allowTypes(classname)
-
-
- def setUnjellyableForClassTree(module, baseClass, prefix=None):
- """
- Set all classes in a module derived from C{baseClass} as copiers for
- a corresponding remote class.
-
- When you have a hierarchy of Copyable (or Cacheable) classes on one
- side, and a mirror structure of Copied (or RemoteCache) classes on the
- other, use this to setUnjellyableForClass all your Copieds for the
- Copyables.
-
- Each copyTag (the \"classname\" argument to getTypeToCopyFor, and
- what the Copyable's getTypeToCopyFor returns) is formed from
- adding a prefix to the Copied's class name. The prefix defaults
- to module.__name__. If you wish the copy tag to consist of solely
- the classname, pass the empty string \'\'.
-
- @param module: a module object from which to pull the Copied classes.
- (passing sys.modules[__name__] might be useful)
-
- @param baseClass: the base class from which all your Copied classes derive.
-
- @param prefix: the string prefixed to classnames to form the
- unjellyableRegistry.
- """
- if prefix is None:
- prefix = module.__name__
-
- if prefix:
- prefix = "%s." % prefix
-
- for name in dir(module):
- loaded = getattr(module, name)
- try:
- yes = issubclass(loaded, baseClass)
- except TypeError:
- "It's not a class."
- else:
- if yes:
- setUnjellyableForClass(f"{prefix}{name}", loaded)
-
-
- def getInstanceState(inst, jellier):
- """
- Utility method to default to 'normal' state rules in serialization.
- """
- if hasattr(inst, "__getstate__"):
- state = inst.__getstate__()
- else:
- state = inst.__dict__
- sxp = jellier.prepare(inst)
- sxp.extend([qual(inst.__class__).encode("utf-8"), jellier.jelly(state)])
- return jellier.preserve(inst, sxp)
-
-
- def setInstanceState(inst, unjellier, jellyList):
- """
- Utility method to default to 'normal' state rules in unserialization.
- """
- state = unjellier.unjelly(jellyList[1])
- if hasattr(inst, "__setstate__"):
- inst.__setstate__(state)
- else:
- inst.__dict__ = state
- return inst
-
-
- class Unpersistable:
- """
- This is an instance of a class that comes back when something couldn't be
- unpersisted.
- """
-
- def __init__(self, reason):
- """
- Initialize an unpersistable object with a descriptive C{reason} string.
- """
- self.reason = reason
-
- def __repr__(self) -> str:
- return "Unpersistable(%s)" % repr(self.reason)
-
-
- @implementer(IJellyable)
- class Jellyable:
- """
- Inherit from me to Jelly yourself directly with the `getStateFor'
- convenience method.
- """
-
- def getStateFor(self, jellier):
- return self.__dict__
-
- def jellyFor(self, jellier):
- """
- @see: L{twisted.spread.interfaces.IJellyable.jellyFor}
- """
- sxp = jellier.prepare(self)
- sxp.extend(
- [
- qual(self.__class__).encode("utf-8"),
- jellier.jelly(self.getStateFor(jellier)),
- ]
- )
- return jellier.preserve(self, sxp)
-
-
- @implementer(IUnjellyable)
- class Unjellyable:
- """
- Inherit from me to Unjelly yourself directly with the
- C{setStateFor} convenience method.
- """
-
- def setStateFor(self, unjellier, state):
- self.__dict__ = state
-
- def unjellyFor(self, unjellier, jellyList):
- """
- Perform the inverse operation of L{Jellyable.jellyFor}.
-
- @see: L{twisted.spread.interfaces.IUnjellyable.unjellyFor}
- """
- state = unjellier.unjelly(jellyList[1])
- self.setStateFor(unjellier, state)
- return self
-
-
- class _Jellier:
- """
- (Internal) This class manages state for a call to jelly()
- """
-
- def __init__(self, taster, persistentStore, invoker):
- """
- Initialize.
- """
- self.taster = taster
- # `preserved' is a dict of previously seen instances.
- self.preserved = {}
- # `cooked' is a dict of previously backreferenced instances to their
- # `ref' lists.
- self.cooked = {}
- self.cooker = {}
- self._ref_id = 1
- self.persistentStore = persistentStore
- self.invoker = invoker
-
- def _cook(self, object):
- """
- (internal) Backreference an object.
-
- Notes on this method for the hapless future maintainer: If I've already
- gone through the prepare/preserve cycle on the specified object (it is
- being referenced after the serializer is \"done with\" it, e.g. this
- reference is NOT circular), the copy-in-place of aList is relevant,
- since the list being modified is the actual, pre-existing jelly
- expression that was returned for that object. If not, it's technically
- superfluous, since the value in self.preserved didn't need to be set,
- but the invariant that self.preserved[id(object)] is a list is
- convenient because that means we don't have to test and create it or
- not create it here, creating fewer code-paths. that's why
- self.preserved is always set to a list.
-
- Sorry that this code is so hard to follow, but Python objects are
- tricky to persist correctly. -glyph
- """
- aList = self.preserved[id(object)]
- newList = copy.copy(aList)
- # make a new reference ID
- refid = self._ref_id
- self._ref_id = self._ref_id + 1
- # replace the old list in-place, so that we don't have to track the
- # previous reference to it.
- aList[:] = [reference_atom, refid, newList]
- self.cooked[id(object)] = [dereference_atom, refid]
- return aList
-
- def prepare(self, object):
- """
- (internal) Create a list for persisting an object to. This will allow
- backreferences to be made internal to the object. (circular
- references).
-
- The reason this needs to happen is that we don't generate an ID for
- every object, so we won't necessarily know which ID the object will
- have in the future. When it is 'cooked' ( see _cook ), it will be
- assigned an ID, and the temporary placeholder list created here will be
- modified in-place to create an expression that gives this object an ID:
- [reference id# [object-jelly]].
- """
-
- # create a placeholder list to be preserved
- self.preserved[id(object)] = []
- # keep a reference to this object around, so it doesn't disappear!
- # (This isn't always necessary, but for cases where the objects are
- # dynamically generated by __getstate__ or getStateToCopyFor calls, it
- # is; id() will return the same value for a different object if it gets
- # garbage collected. This may be optimized later.)
- self.cooker[id(object)] = object
- return []
-
- def preserve(self, object, sexp):
- """
- (internal) Mark an object's persistent list for later referral.
- """
- # if I've been cooked in the meanwhile,
- if id(object) in self.cooked:
- # replace the placeholder empty list with the real one
- self.preserved[id(object)][2] = sexp
- # but give this one back.
- sexp = self.preserved[id(object)]
- else:
- self.preserved[id(object)] = sexp
- return sexp
-
- def _checkMutable(self, obj):
- objId = id(obj)
- if objId in self.cooked:
- return self.cooked[objId]
- if objId in self.preserved:
- self._cook(obj)
- return self.cooked[objId]
-
- def jelly(self, obj):
- if isinstance(obj, Jellyable):
- preRef = self._checkMutable(obj)
- if preRef:
- return preRef
- return obj.jellyFor(self)
- objType = type(obj)
- if self.taster.isTypeAllowed(qual(objType).encode("utf-8")):
- # "Immutable" Types
- if objType in (bytes, int, float):
- return obj
- elif isinstance(obj, types.MethodType):
- aSelf = obj.__self__
- aFunc = obj.__func__
- aClass = aSelf.__class__
- return [
- b"method",
- aFunc.__name__,
- self.jelly(aSelf),
- self.jelly(aClass),
- ]
- elif objType is str:
- return [b"unicode", obj.encode("UTF-8")]
- elif isinstance(obj, type(None)):
- return [b"None"]
- elif isinstance(obj, types.FunctionType):
- return [b"function", obj.__module__ + "." + obj.__qualname__]
- elif isinstance(obj, types.ModuleType):
- return [b"module", obj.__name__]
- elif objType is bool:
- return [b"boolean", obj and b"true" or b"false"]
- elif objType is datetime.datetime:
- if obj.tzinfo:
- raise NotImplementedError(
- "Currently can't jelly datetime objects with tzinfo"
- )
- return [
- b"datetime",
- " ".join(
- [
- str(x)
- for x in (
- obj.year,
- obj.month,
- obj.day,
- obj.hour,
- obj.minute,
- obj.second,
- obj.microsecond,
- )
- ]
- ).encode("utf-8"),
- ]
- elif objType is datetime.time:
- if obj.tzinfo:
- raise NotImplementedError(
- "Currently can't jelly datetime objects with tzinfo"
- )
- return [
- b"time",
- " ".join(
- [
- str(x)
- for x in (obj.hour, obj.minute, obj.second, obj.microsecond)
- ]
- ).encode("utf-8"),
- ]
- elif objType is datetime.date:
- return [
- b"date",
- " ".join([str(x) for x in (obj.year, obj.month, obj.day)]).encode(
- "utf-8"
- ),
- ]
- elif objType is datetime.timedelta:
- return [
- b"timedelta",
- " ".join(
- [str(x) for x in (obj.days, obj.seconds, obj.microseconds)]
- ).encode("utf-8"),
- ]
- elif issubclass(objType, type):
- return [b"class", qual(obj).encode("utf-8")]
- elif objType is decimal.Decimal:
- return self.jelly_decimal(obj)
- else:
- preRef = self._checkMutable(obj)
- if preRef:
- return preRef
- # "Mutable" Types
- sxp = self.prepare(obj)
- if objType is list:
- sxp.extend(self._jellyIterable(list_atom, obj))
- elif objType is tuple:
- sxp.extend(self._jellyIterable(tuple_atom, obj))
- elif objType in DictTypes:
- sxp.append(dictionary_atom)
- for key, val in obj.items():
- sxp.append([self.jelly(key), self.jelly(val)])
- elif objType is set:
- sxp.extend(self._jellyIterable(set_atom, obj))
- elif objType is frozenset:
- sxp.extend(self._jellyIterable(frozenset_atom, obj))
- else:
- className = qual(obj.__class__).encode("utf-8")
- persistent = None
- if self.persistentStore:
- persistent = self.persistentStore(obj, self)
- if persistent is not None:
- sxp.append(persistent_atom)
- sxp.append(persistent)
- elif self.taster.isClassAllowed(obj.__class__):
- sxp.append(className)
- if hasattr(obj, "__getstate__"):
- state = obj.__getstate__()
- else:
- state = obj.__dict__
- sxp.append(self.jelly(state))
- else:
- self.unpersistable(
- "instance of class %s deemed insecure"
- % qual(obj.__class__),
- sxp,
- )
- return self.preserve(obj, sxp)
- else:
- raise InsecureJelly(f"Type not allowed for object: {objType} {obj}")
-
- def _jellyIterable(self, atom, obj):
- """
- Jelly an iterable object.
-
- @param atom: the identifier atom of the object.
- @type atom: C{str}
-
- @param obj: any iterable object.
- @type obj: C{iterable}
-
- @return: a generator of jellied data.
- @rtype: C{generator}
- """
- yield atom
- for item in obj:
- yield self.jelly(item)
-
- def jelly_decimal(self, d):
- """
- Jelly a decimal object.
-
- @param d: a decimal object to serialize.
- @type d: C{decimal.Decimal}
-
- @return: jelly for the decimal object.
- @rtype: C{list}
- """
- sign, guts, exponent = d.as_tuple()
- value = reduce(lambda left, right: left * 10 + right, guts)
- if sign:
- value = -value
- return [b"decimal", value, exponent]
-
- def unpersistable(self, reason, sxp=None):
- """
- (internal) Returns an sexp: (unpersistable "reason"). Utility method
- for making note that a particular object could not be serialized.
- """
- if sxp is None:
- sxp = []
- sxp.append(unpersistable_atom)
- if isinstance(reason, str):
- reason = reason.encode("utf-8")
- sxp.append(reason)
- return sxp
-
-
- class _Unjellier:
- def __init__(self, taster, persistentLoad, invoker):
- self.taster = taster
- self.persistentLoad = persistentLoad
- self.references = {}
- self.postCallbacks = []
- self.invoker = invoker
-
- def unjellyFull(self, obj):
- o = self.unjelly(obj)
- for m in self.postCallbacks:
- m()
- return o
-
- def _maybePostUnjelly(self, unjellied):
- """
- If the given object has support for the C{postUnjelly} hook, set it up
- to be called at the end of deserialization.
-
- @param unjellied: an object that has already been unjellied.
-
- @return: C{unjellied}
- """
- if hasattr(unjellied, "postUnjelly"):
- self.postCallbacks.append(unjellied.postUnjelly)
- return unjellied
-
- def unjelly(self, obj):
- if type(obj) is not list:
- return obj
- jelTypeBytes = obj[0]
- if not self.taster.isTypeAllowed(jelTypeBytes):
- raise InsecureJelly(jelTypeBytes)
- regClass = unjellyableRegistry.get(jelTypeBytes)
- if regClass is not None:
- method = getattr(_createBlank(regClass), "unjellyFor", regClass)
- return self._maybePostUnjelly(method(self, obj))
- regFactory = unjellyableFactoryRegistry.get(jelTypeBytes)
- if regFactory is not None:
- return self._maybePostUnjelly(regFactory(self.unjelly(obj[1])))
-
- jelTypeText = nativeString(jelTypeBytes)
- thunk = getattr(self, "_unjelly_%s" % jelTypeText, None)
- if thunk is not None:
- return thunk(obj[1:])
- else:
- nameSplit = jelTypeText.split(".")
- modName = ".".join(nameSplit[:-1])
- if not self.taster.isModuleAllowed(modName):
- raise InsecureJelly(
- f"Module {modName} not allowed (in type {jelTypeText})."
- )
- clz = namedObject(jelTypeText)
- if not self.taster.isClassAllowed(clz):
- raise InsecureJelly("Class %s not allowed." % jelTypeText)
- return self._genericUnjelly(clz, obj[1])
-
- def _genericUnjelly(self, cls, state):
- """
- Unjelly a type for which no specific unjellier is registered, but which
- is nonetheless allowed.
-
- @param cls: the class of the instance we are unjellying.
- @type cls: L{type}
-
- @param state: The jellied representation of the object's state; its
- C{__dict__} unless it has a C{__setstate__} that takes something
- else.
- @type state: L{list}
-
- @return: the new, unjellied instance.
- """
- return self._maybePostUnjelly(_newInstance(cls, self.unjelly(state)))
-
- def _unjelly_None(self, exp):
- return None
-
- def _unjelly_unicode(self, exp):
- return str(exp[0], "UTF-8")
-
- def _unjelly_decimal(self, exp):
- """
- Unjelly decimal objects.
- """
- value = exp[0]
- exponent = exp[1]
- if value < 0:
- sign = 1
- else:
- sign = 0
- guts = decimal.Decimal(value).as_tuple()[1]
- return decimal.Decimal((sign, guts, exponent))
-
- def _unjelly_boolean(self, exp):
- assert exp[0] in (b"true", b"false")
- return exp[0] == b"true"
-
- def _unjelly_datetime(self, exp):
- return datetime.datetime(*map(int, exp[0].split()))
-
- def _unjelly_date(self, exp):
- return datetime.date(*map(int, exp[0].split()))
-
- def _unjelly_time(self, exp):
- return datetime.time(*map(int, exp[0].split()))
-
- def _unjelly_timedelta(self, exp):
- days, seconds, microseconds = map(int, exp[0].split())
- return datetime.timedelta(days=days, seconds=seconds, microseconds=microseconds)
-
- def unjellyInto(self, obj, loc, jel):
- o = self.unjelly(jel)
- if isinstance(o, NotKnown):
- o.addDependant(obj, loc)
- obj[loc] = o
- return o
-
- def _unjelly_dereference(self, lst):
- refid = lst[0]
- x = self.references.get(refid)
- if x is not None:
- return x
- der = _Dereference(refid)
- self.references[refid] = der
- return der
-
- def _unjelly_reference(self, lst):
- refid = lst[0]
- exp = lst[1]
- o = self.unjelly(exp)
- ref = self.references.get(refid)
- if ref is None:
- self.references[refid] = o
- elif isinstance(ref, NotKnown):
- ref.resolveDependants(o)
- self.references[refid] = o
- else:
- assert 0, "Multiple references with same ID!"
- return o
-
- def _unjelly_tuple(self, lst):
- l = list(range(len(lst)))
- finished = 1
- for elem in l:
- if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown):
- finished = 0
- if finished:
- return tuple(l)
- else:
- return _Tuple(l)
-
- def _unjelly_list(self, lst):
- l = list(range(len(lst)))
- for elem in l:
- self.unjellyInto(l, elem, lst[elem])
- return l
-
- def _unjellySetOrFrozenset(self, lst, containerType):
- """
- Helper method to unjelly set or frozenset.
-
- @param lst: the content of the set.
- @type lst: C{list}
-
- @param containerType: the type of C{set} to use.
- """
- l = list(range(len(lst)))
- finished = True
- for elem in l:
- data = self.unjellyInto(l, elem, lst[elem])
- if isinstance(data, NotKnown):
- finished = False
- if not finished:
- return _Container(l, containerType)
- else:
- return containerType(l)
-
- def _unjelly_set(self, lst):
- """
- Unjelly set using the C{set} builtin.
- """
- return self._unjellySetOrFrozenset(lst, set)
-
- def _unjelly_frozenset(self, lst):
- """
- Unjelly frozenset using the C{frozenset} builtin.
- """
- return self._unjellySetOrFrozenset(lst, frozenset)
-
- def _unjelly_dictionary(self, lst):
- d = {}
- for k, v in lst:
- kvd = _DictKeyAndValue(d)
- self.unjellyInto(kvd, 0, k)
- self.unjellyInto(kvd, 1, v)
- return d
-
- def _unjelly_module(self, rest):
- moduleName = nativeString(rest[0])
- if type(moduleName) != str:
- raise InsecureJelly("Attempted to unjelly a module with a non-string name.")
- if not self.taster.isModuleAllowed(moduleName):
- raise InsecureJelly(f"Attempted to unjelly module named {moduleName!r}")
- mod = __import__(moduleName, {}, {}, "x")
- return mod
-
- def _unjelly_class(self, rest):
- cname = nativeString(rest[0])
- clist = cname.split(nativeString("."))
- modName = nativeString(".").join(clist[:-1])
- if not self.taster.isModuleAllowed(modName):
- raise InsecureJelly("module %s not allowed" % modName)
- klaus = namedObject(cname)
- objType = type(klaus)
- if objType is not type:
- raise InsecureJelly(
- "class %r unjellied to something that isn't a class: %r"
- % (cname, klaus)
- )
- if not self.taster.isClassAllowed(klaus):
- raise InsecureJelly("class not allowed: %s" % qual(klaus))
- return klaus
-
- def _unjelly_function(self, rest):
- fname = nativeString(rest[0])
- modSplit = fname.split(nativeString("."))
- modName = nativeString(".").join(modSplit[:-1])
- if not self.taster.isModuleAllowed(modName):
- raise InsecureJelly("Module not allowed: %s" % modName)
- # XXX do I need an isFunctionAllowed?
- function = namedAny(fname)
- return function
-
- def _unjelly_persistent(self, rest):
- if self.persistentLoad:
- pload = self.persistentLoad(rest[0], self)
- return pload
- else:
- return Unpersistable("Persistent callback not found")
-
- def _unjelly_instance(self, rest):
- """
- (internal) Unjelly an instance.
-
- Called to handle the deprecated I{instance} token.
-
- @param rest: The s-expression representing the instance.
-
- @return: The unjellied instance.
- """
- warnings.warn_explicit(
- "Unjelly support for the instance atom is deprecated since "
- "Twisted 15.0.0. Upgrade peer for modern instance support.",
- category=DeprecationWarning,
- filename="",
- lineno=0,
- )
-
- clz = self.unjelly(rest[0])
- return self._genericUnjelly(clz, rest[1])
-
- def _unjelly_unpersistable(self, rest):
- return Unpersistable(f"Unpersistable data: {rest[0]}")
-
- def _unjelly_method(self, rest):
- """
- (internal) Unjelly a method.
- """
- im_name = rest[0]
- im_self = self.unjelly(rest[1])
- im_class = self.unjelly(rest[2])
- if not isinstance(im_class, type):
- raise InsecureJelly("Method found with non-class class.")
- if im_name in im_class.__dict__:
- if im_self is None:
- im = getattr(im_class, im_name)
- elif isinstance(im_self, NotKnown):
- im = _InstanceMethod(im_name, im_self, im_class)
- else:
- im = types.MethodType(
- im_class.__dict__[im_name], im_self, *([im_class] * (False))
- )
- else:
- raise TypeError("instance method changed")
- return im
-
-
- #### Published Interface.
-
-
- class InsecureJelly(Exception):
- """
- This exception will be raised when a jelly is deemed `insecure'; e.g. it
- contains a type, class, or module disallowed by the specified `taster'
- """
-
-
- class DummySecurityOptions:
- """
- DummySecurityOptions() -> insecure security options
- Dummy security options -- this class will allow anything.
- """
-
- def isModuleAllowed(self, moduleName):
- """
- DummySecurityOptions.isModuleAllowed(moduleName) -> boolean
- returns 1 if a module by that name is allowed, 0 otherwise
- """
- return 1
-
- def isClassAllowed(self, klass):
- """
- DummySecurityOptions.isClassAllowed(class) -> boolean
- Assumes the module has already been allowed. Returns 1 if the given
- class is allowed, 0 otherwise.
- """
- return 1
-
- def isTypeAllowed(self, typeName):
- """
- DummySecurityOptions.isTypeAllowed(typeName) -> boolean
- Returns 1 if the given type is allowed, 0 otherwise.
- """
- return 1
-
-
- class SecurityOptions:
- """
- This will by default disallow everything, except for 'none'.
- """
-
- basicTypes = [
- "dictionary",
- "list",
- "tuple",
- "reference",
- "dereference",
- "unpersistable",
- "persistent",
- "long_int",
- "long",
- "dict",
- ]
-
- def __init__(self):
- """
- SecurityOptions() initialize.
- """
- # I don't believe any of these types can ever pose a security hazard,
- # except perhaps "reference"...
- self.allowedTypes = {
- b"None": 1,
- b"bool": 1,
- b"boolean": 1,
- b"string": 1,
- b"str": 1,
- b"int": 1,
- b"float": 1,
- b"datetime": 1,
- b"time": 1,
- b"date": 1,
- b"timedelta": 1,
- b"NoneType": 1,
- b"unicode": 1,
- b"decimal": 1,
- b"set": 1,
- b"frozenset": 1,
- }
- self.allowedModules = {}
- self.allowedClasses = {}
-
- def allowBasicTypes(self):
- """
- Allow all `basic' types. (Dictionary and list. Int, string, and float
- are implicitly allowed.)
- """
- self.allowTypes(*self.basicTypes)
-
- def allowTypes(self, *types):
- """
- SecurityOptions.allowTypes(typeString): Allow a particular type, by its
- name.
- """
- for typ in types:
- if isinstance(typ, str):
- typ = typ.encode("utf-8")
- if not isinstance(typ, bytes):
- typ = qual(typ)
- self.allowedTypes[typ] = 1
-
- def allowInstancesOf(self, *classes):
- """
- SecurityOptions.allowInstances(klass, klass, ...): allow instances
- of the specified classes
-
- This will also allow the 'instance', 'class' (renamed 'classobj' in
- Python 2.3), and 'module' types, as well as basic types.
- """
- self.allowBasicTypes()
- self.allowTypes("instance", "class", "classobj", "module")
- for klass in classes:
- self.allowTypes(qual(klass))
- self.allowModules(klass.__module__)
- self.allowedClasses[klass] = 1
-
- def allowModules(self, *modules):
- """
- SecurityOptions.allowModules(module, module, ...): allow modules by
- name. This will also allow the 'module' type.
- """
- for module in modules:
- if type(module) == types.ModuleType:
- module = module.__name__
-
- if not isinstance(module, bytes):
- module = module.encode("utf-8")
-
- self.allowedModules[module] = 1
-
- def isModuleAllowed(self, moduleName):
- """
- SecurityOptions.isModuleAllowed(moduleName) -> boolean
- returns 1 if a module by that name is allowed, 0 otherwise
- """
- if not isinstance(moduleName, bytes):
- moduleName = moduleName.encode("utf-8")
-
- return moduleName in self.allowedModules
-
- def isClassAllowed(self, klass):
- """
- SecurityOptions.isClassAllowed(class) -> boolean
- Assumes the module has already been allowed. Returns 1 if the given
- class is allowed, 0 otherwise.
- """
- return klass in self.allowedClasses
-
- def isTypeAllowed(self, typeName):
- """
- SecurityOptions.isTypeAllowed(typeName) -> boolean
- Returns 1 if the given type is allowed, 0 otherwise.
- """
- if not isinstance(typeName, bytes):
- typeName = typeName.encode("utf-8")
-
- return typeName in self.allowedTypes or b"." in typeName
-
-
- globalSecurity = SecurityOptions()
- globalSecurity.allowBasicTypes()
-
-
- def jelly(object, taster=DummySecurityOptions(), persistentStore=None, invoker=None):
- """
- Serialize to s-expression.
-
- Returns a list which is the serialized representation of an object. An
- optional 'taster' argument takes a SecurityOptions and will mark any
- insecure objects as unpersistable rather than serializing them.
- """
- return _Jellier(taster, persistentStore, invoker).jelly(object)
-
-
- def unjelly(sexp, taster=DummySecurityOptions(), persistentLoad=None, invoker=None):
- """
- Unserialize from s-expression.
-
- Takes a list that was the result from a call to jelly() and unserializes
- an arbitrary object from it. The optional 'taster' argument, an instance
- of SecurityOptions, will cause an InsecureJelly exception to be raised if a
- disallowed type, module, or class attempted to unserialize.
- """
- return _Unjellier(taster, persistentLoad, invoker).unjellyFull(sexp)
|