Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

jelly.py 35KB

1 year ago
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091
  1. # -*- test-case-name: twisted.spread.test.test_jelly -*-
  2. # Copyright (c) Twisted Matrix Laboratories.
  3. # See LICENSE for details.
  4. """
  5. S-expression-based persistence of python objects.
  6. It does something very much like L{Pickle<pickle>}; however, pickle's main goal
  7. seems to be efficiency (both in space and time); jelly's main goals are
  8. security, human readability, and portability to other environments.
  9. This is how Jelly converts various objects to s-expressions.
  10. Boolean::
  11. True --> ['boolean', 'true']
  12. Integer::
  13. 1 --> 1
  14. List::
  15. [1, 2] --> ['list', 1, 2]
  16. String::
  17. \"hello\" --> \"hello\"
  18. Float::
  19. 2.3 --> 2.3
  20. Dictionary::
  21. {'a': 1, 'b': 'c'} --> ['dictionary', ['b', 'c'], ['a', 1]]
  22. Module::
  23. UserString --> ['module', 'UserString']
  24. Class::
  25. UserString.UserString --> ['class', ['module', 'UserString'], 'UserString']
  26. Function::
  27. string.join --> ['function', 'join', ['module', 'string']]
  28. Instance: s is an instance of UserString.UserString, with a __dict__
  29. {'data': 'hello'}::
  30. [\"UserString.UserString\", ['dictionary', ['data', 'hello']]]
  31. Class Method: UserString.UserString.center::
  32. ['method', 'center', ['None'], ['class', ['module', 'UserString'],
  33. 'UserString']]
  34. Instance Method: s.center, where s is an instance of UserString.UserString::
  35. ['method', 'center', ['instance', ['reference', 1, ['class',
  36. ['module', 'UserString'], 'UserString']], ['dictionary', ['data', 'd']]],
  37. ['dereference', 1]]
  38. The Python 2.x C{sets.Set} and C{sets.ImmutableSet} classes are
  39. serialized to the same thing as the builtin C{set} and C{frozenset}
  40. classes. (This is only relevant if you are communicating with a
  41. version of jelly running on an older version of Python.)
  42. @author: Glyph Lefkowitz
  43. """
  44. import copy
  45. import datetime
  46. import decimal
  47. # System Imports
  48. import types
  49. import warnings
  50. from functools import reduce
  51. from zope.interface import implementer
  52. from incremental import Version
  53. from twisted.persisted.crefutil import (
  54. NotKnown,
  55. _Container,
  56. _Dereference,
  57. _DictKeyAndValue,
  58. _InstanceMethod,
  59. _Tuple,
  60. )
  61. # Twisted Imports
  62. from twisted.python.compat import nativeString
  63. from twisted.python.deprecate import deprecatedModuleAttribute
  64. from twisted.python.reflect import namedAny, namedObject, qual
  65. from twisted.spread.interfaces import IJellyable, IUnjellyable
  66. DictTypes = (dict,)
  67. None_atom = b"None" # N
  68. # code
  69. class_atom = b"class" # c
  70. module_atom = b"module" # m
  71. function_atom = b"function" # f
  72. # references
  73. dereference_atom = b"dereference" # D
  74. persistent_atom = b"persistent" # p
  75. reference_atom = b"reference" # r
  76. # mutable collections
  77. dictionary_atom = b"dictionary" # d
  78. list_atom = b"list" # l
  79. set_atom = b"set"
  80. # immutable collections
  81. # (assignment to __dict__ and __class__ still might go away!)
  82. tuple_atom = b"tuple" # t
  83. instance_atom = b"instance" # i
  84. frozenset_atom = b"frozenset"
  85. deprecatedModuleAttribute(
  86. Version("Twisted", 15, 0, 0),
  87. "instance_atom is unused within Twisted.",
  88. "twisted.spread.jelly",
  89. "instance_atom",
  90. )
  91. # errors
  92. unpersistable_atom = b"unpersistable" # u
  93. unjellyableRegistry = {}
  94. unjellyableFactoryRegistry = {}
  95. def _createBlank(cls):
  96. """
  97. Given an object, if that object is a type, return a new, blank instance
  98. of that type which has not had C{__init__} called on it. If the object
  99. is not a type, return L{None}.
  100. @param cls: The type (or class) to create an instance of.
  101. @type cls: L{type} or something else that cannot be
  102. instantiated.
  103. @return: a new blank instance or L{None} if C{cls} is not a class or type.
  104. """
  105. if isinstance(cls, type):
  106. return cls.__new__(cls)
  107. def _newInstance(cls, state):
  108. """
  109. Make a new instance of a class without calling its __init__ method.
  110. @param state: A C{dict} used to update C{inst.__dict__} either directly or
  111. via C{__setstate__}, if available.
  112. @return: A new instance of C{cls}.
  113. """
  114. instance = _createBlank(cls)
  115. def defaultSetter(state):
  116. instance.__dict__ = state
  117. setter = getattr(instance, "__setstate__", defaultSetter)
  118. setter(state)
  119. return instance
  120. def _maybeClass(classnamep):
  121. isObject = isinstance(classnamep, type)
  122. if isObject:
  123. classnamep = qual(classnamep)
  124. if not isinstance(classnamep, bytes):
  125. classnamep = classnamep.encode("utf-8")
  126. return classnamep
  127. def setUnjellyableForClass(classname, unjellyable):
  128. """
  129. Set which local class will represent a remote type.
  130. If you have written a Copyable class that you expect your client to be
  131. receiving, write a local "copy" class to represent it, then call::
  132. jellier.setUnjellyableForClass('module.package.Class', MyCopier).
  133. Call this at the module level immediately after its class
  134. definition. MyCopier should be a subclass of RemoteCopy.
  135. The classname may be a special tag returned by
  136. 'Copyable.getTypeToCopyFor' rather than an actual classname.
  137. This call is also for cached classes, since there will be no
  138. overlap. The rules are the same.
  139. """
  140. global unjellyableRegistry
  141. classname = _maybeClass(classname)
  142. unjellyableRegistry[classname] = unjellyable
  143. globalSecurity.allowTypes(classname)
  144. def setUnjellyableFactoryForClass(classname, copyFactory):
  145. """
  146. Set the factory to construct a remote instance of a type::
  147. jellier.setUnjellyableFactoryForClass('module.package.Class', MyFactory)
  148. Call this at the module level immediately after its class definition.
  149. C{copyFactory} should return an instance or subclass of
  150. L{RemoteCopy<pb.RemoteCopy>}.
  151. Similar to L{setUnjellyableForClass} except it uses a factory instead
  152. of creating an instance.
  153. """
  154. global unjellyableFactoryRegistry
  155. classname = _maybeClass(classname)
  156. unjellyableFactoryRegistry[classname] = copyFactory
  157. globalSecurity.allowTypes(classname)
  158. def setUnjellyableForClassTree(module, baseClass, prefix=None):
  159. """
  160. Set all classes in a module derived from C{baseClass} as copiers for
  161. a corresponding remote class.
  162. When you have a hierarchy of Copyable (or Cacheable) classes on one
  163. side, and a mirror structure of Copied (or RemoteCache) classes on the
  164. other, use this to setUnjellyableForClass all your Copieds for the
  165. Copyables.
  166. Each copyTag (the \"classname\" argument to getTypeToCopyFor, and
  167. what the Copyable's getTypeToCopyFor returns) is formed from
  168. adding a prefix to the Copied's class name. The prefix defaults
  169. to module.__name__. If you wish the copy tag to consist of solely
  170. the classname, pass the empty string \'\'.
  171. @param module: a module object from which to pull the Copied classes.
  172. (passing sys.modules[__name__] might be useful)
  173. @param baseClass: the base class from which all your Copied classes derive.
  174. @param prefix: the string prefixed to classnames to form the
  175. unjellyableRegistry.
  176. """
  177. if prefix is None:
  178. prefix = module.__name__
  179. if prefix:
  180. prefix = "%s." % prefix
  181. for name in dir(module):
  182. loaded = getattr(module, name)
  183. try:
  184. yes = issubclass(loaded, baseClass)
  185. except TypeError:
  186. "It's not a class."
  187. else:
  188. if yes:
  189. setUnjellyableForClass(f"{prefix}{name}", loaded)
  190. def getInstanceState(inst, jellier):
  191. """
  192. Utility method to default to 'normal' state rules in serialization.
  193. """
  194. if hasattr(inst, "__getstate__"):
  195. state = inst.__getstate__()
  196. else:
  197. state = inst.__dict__
  198. sxp = jellier.prepare(inst)
  199. sxp.extend([qual(inst.__class__).encode("utf-8"), jellier.jelly(state)])
  200. return jellier.preserve(inst, sxp)
  201. def setInstanceState(inst, unjellier, jellyList):
  202. """
  203. Utility method to default to 'normal' state rules in unserialization.
  204. """
  205. state = unjellier.unjelly(jellyList[1])
  206. if hasattr(inst, "__setstate__"):
  207. inst.__setstate__(state)
  208. else:
  209. inst.__dict__ = state
  210. return inst
  211. class Unpersistable:
  212. """
  213. This is an instance of a class that comes back when something couldn't be
  214. unpersisted.
  215. """
  216. def __init__(self, reason):
  217. """
  218. Initialize an unpersistable object with a descriptive C{reason} string.
  219. """
  220. self.reason = reason
  221. def __repr__(self) -> str:
  222. return "Unpersistable(%s)" % repr(self.reason)
  223. @implementer(IJellyable)
  224. class Jellyable:
  225. """
  226. Inherit from me to Jelly yourself directly with the `getStateFor'
  227. convenience method.
  228. """
  229. def getStateFor(self, jellier):
  230. return self.__dict__
  231. def jellyFor(self, jellier):
  232. """
  233. @see: L{twisted.spread.interfaces.IJellyable.jellyFor}
  234. """
  235. sxp = jellier.prepare(self)
  236. sxp.extend(
  237. [
  238. qual(self.__class__).encode("utf-8"),
  239. jellier.jelly(self.getStateFor(jellier)),
  240. ]
  241. )
  242. return jellier.preserve(self, sxp)
  243. @implementer(IUnjellyable)
  244. class Unjellyable:
  245. """
  246. Inherit from me to Unjelly yourself directly with the
  247. C{setStateFor} convenience method.
  248. """
  249. def setStateFor(self, unjellier, state):
  250. self.__dict__ = state
  251. def unjellyFor(self, unjellier, jellyList):
  252. """
  253. Perform the inverse operation of L{Jellyable.jellyFor}.
  254. @see: L{twisted.spread.interfaces.IUnjellyable.unjellyFor}
  255. """
  256. state = unjellier.unjelly(jellyList[1])
  257. self.setStateFor(unjellier, state)
  258. return self
  259. class _Jellier:
  260. """
  261. (Internal) This class manages state for a call to jelly()
  262. """
  263. def __init__(self, taster, persistentStore, invoker):
  264. """
  265. Initialize.
  266. """
  267. self.taster = taster
  268. # `preserved' is a dict of previously seen instances.
  269. self.preserved = {}
  270. # `cooked' is a dict of previously backreferenced instances to their
  271. # `ref' lists.
  272. self.cooked = {}
  273. self.cooker = {}
  274. self._ref_id = 1
  275. self.persistentStore = persistentStore
  276. self.invoker = invoker
  277. def _cook(self, object):
  278. """
  279. (internal) Backreference an object.
  280. Notes on this method for the hapless future maintainer: If I've already
  281. gone through the prepare/preserve cycle on the specified object (it is
  282. being referenced after the serializer is \"done with\" it, e.g. this
  283. reference is NOT circular), the copy-in-place of aList is relevant,
  284. since the list being modified is the actual, pre-existing jelly
  285. expression that was returned for that object. If not, it's technically
  286. superfluous, since the value in self.preserved didn't need to be set,
  287. but the invariant that self.preserved[id(object)] is a list is
  288. convenient because that means we don't have to test and create it or
  289. not create it here, creating fewer code-paths. that's why
  290. self.preserved is always set to a list.
  291. Sorry that this code is so hard to follow, but Python objects are
  292. tricky to persist correctly. -glyph
  293. """
  294. aList = self.preserved[id(object)]
  295. newList = copy.copy(aList)
  296. # make a new reference ID
  297. refid = self._ref_id
  298. self._ref_id = self._ref_id + 1
  299. # replace the old list in-place, so that we don't have to track the
  300. # previous reference to it.
  301. aList[:] = [reference_atom, refid, newList]
  302. self.cooked[id(object)] = [dereference_atom, refid]
  303. return aList
  304. def prepare(self, object):
  305. """
  306. (internal) Create a list for persisting an object to. This will allow
  307. backreferences to be made internal to the object. (circular
  308. references).
  309. The reason this needs to happen is that we don't generate an ID for
  310. every object, so we won't necessarily know which ID the object will
  311. have in the future. When it is 'cooked' ( see _cook ), it will be
  312. assigned an ID, and the temporary placeholder list created here will be
  313. modified in-place to create an expression that gives this object an ID:
  314. [reference id# [object-jelly]].
  315. """
  316. # create a placeholder list to be preserved
  317. self.preserved[id(object)] = []
  318. # keep a reference to this object around, so it doesn't disappear!
  319. # (This isn't always necessary, but for cases where the objects are
  320. # dynamically generated by __getstate__ or getStateToCopyFor calls, it
  321. # is; id() will return the same value for a different object if it gets
  322. # garbage collected. This may be optimized later.)
  323. self.cooker[id(object)] = object
  324. return []
  325. def preserve(self, object, sexp):
  326. """
  327. (internal) Mark an object's persistent list for later referral.
  328. """
  329. # if I've been cooked in the meanwhile,
  330. if id(object) in self.cooked:
  331. # replace the placeholder empty list with the real one
  332. self.preserved[id(object)][2] = sexp
  333. # but give this one back.
  334. sexp = self.preserved[id(object)]
  335. else:
  336. self.preserved[id(object)] = sexp
  337. return sexp
  338. def _checkMutable(self, obj):
  339. objId = id(obj)
  340. if objId in self.cooked:
  341. return self.cooked[objId]
  342. if objId in self.preserved:
  343. self._cook(obj)
  344. return self.cooked[objId]
  345. def jelly(self, obj):
  346. if isinstance(obj, Jellyable):
  347. preRef = self._checkMutable(obj)
  348. if preRef:
  349. return preRef
  350. return obj.jellyFor(self)
  351. objType = type(obj)
  352. if self.taster.isTypeAllowed(qual(objType).encode("utf-8")):
  353. # "Immutable" Types
  354. if objType in (bytes, int, float):
  355. return obj
  356. elif isinstance(obj, types.MethodType):
  357. aSelf = obj.__self__
  358. aFunc = obj.__func__
  359. aClass = aSelf.__class__
  360. return [
  361. b"method",
  362. aFunc.__name__,
  363. self.jelly(aSelf),
  364. self.jelly(aClass),
  365. ]
  366. elif objType is str:
  367. return [b"unicode", obj.encode("UTF-8")]
  368. elif isinstance(obj, type(None)):
  369. return [b"None"]
  370. elif isinstance(obj, types.FunctionType):
  371. return [b"function", obj.__module__ + "." + obj.__qualname__]
  372. elif isinstance(obj, types.ModuleType):
  373. return [b"module", obj.__name__]
  374. elif objType is bool:
  375. return [b"boolean", obj and b"true" or b"false"]
  376. elif objType is datetime.datetime:
  377. if obj.tzinfo:
  378. raise NotImplementedError(
  379. "Currently can't jelly datetime objects with tzinfo"
  380. )
  381. return [
  382. b"datetime",
  383. " ".join(
  384. [
  385. str(x)
  386. for x in (
  387. obj.year,
  388. obj.month,
  389. obj.day,
  390. obj.hour,
  391. obj.minute,
  392. obj.second,
  393. obj.microsecond,
  394. )
  395. ]
  396. ).encode("utf-8"),
  397. ]
  398. elif objType is datetime.time:
  399. if obj.tzinfo:
  400. raise NotImplementedError(
  401. "Currently can't jelly datetime objects with tzinfo"
  402. )
  403. return [
  404. b"time",
  405. " ".join(
  406. [
  407. str(x)
  408. for x in (obj.hour, obj.minute, obj.second, obj.microsecond)
  409. ]
  410. ).encode("utf-8"),
  411. ]
  412. elif objType is datetime.date:
  413. return [
  414. b"date",
  415. " ".join([str(x) for x in (obj.year, obj.month, obj.day)]).encode(
  416. "utf-8"
  417. ),
  418. ]
  419. elif objType is datetime.timedelta:
  420. return [
  421. b"timedelta",
  422. " ".join(
  423. [str(x) for x in (obj.days, obj.seconds, obj.microseconds)]
  424. ).encode("utf-8"),
  425. ]
  426. elif issubclass(objType, type):
  427. return [b"class", qual(obj).encode("utf-8")]
  428. elif objType is decimal.Decimal:
  429. return self.jelly_decimal(obj)
  430. else:
  431. preRef = self._checkMutable(obj)
  432. if preRef:
  433. return preRef
  434. # "Mutable" Types
  435. sxp = self.prepare(obj)
  436. if objType is list:
  437. sxp.extend(self._jellyIterable(list_atom, obj))
  438. elif objType is tuple:
  439. sxp.extend(self._jellyIterable(tuple_atom, obj))
  440. elif objType in DictTypes:
  441. sxp.append(dictionary_atom)
  442. for key, val in obj.items():
  443. sxp.append([self.jelly(key), self.jelly(val)])
  444. elif objType is set:
  445. sxp.extend(self._jellyIterable(set_atom, obj))
  446. elif objType is frozenset:
  447. sxp.extend(self._jellyIterable(frozenset_atom, obj))
  448. else:
  449. className = qual(obj.__class__).encode("utf-8")
  450. persistent = None
  451. if self.persistentStore:
  452. persistent = self.persistentStore(obj, self)
  453. if persistent is not None:
  454. sxp.append(persistent_atom)
  455. sxp.append(persistent)
  456. elif self.taster.isClassAllowed(obj.__class__):
  457. sxp.append(className)
  458. if hasattr(obj, "__getstate__"):
  459. state = obj.__getstate__()
  460. else:
  461. state = obj.__dict__
  462. sxp.append(self.jelly(state))
  463. else:
  464. self.unpersistable(
  465. "instance of class %s deemed insecure"
  466. % qual(obj.__class__),
  467. sxp,
  468. )
  469. return self.preserve(obj, sxp)
  470. else:
  471. raise InsecureJelly(f"Type not allowed for object: {objType} {obj}")
  472. def _jellyIterable(self, atom, obj):
  473. """
  474. Jelly an iterable object.
  475. @param atom: the identifier atom of the object.
  476. @type atom: C{str}
  477. @param obj: any iterable object.
  478. @type obj: C{iterable}
  479. @return: a generator of jellied data.
  480. @rtype: C{generator}
  481. """
  482. yield atom
  483. for item in obj:
  484. yield self.jelly(item)
  485. def jelly_decimal(self, d):
  486. """
  487. Jelly a decimal object.
  488. @param d: a decimal object to serialize.
  489. @type d: C{decimal.Decimal}
  490. @return: jelly for the decimal object.
  491. @rtype: C{list}
  492. """
  493. sign, guts, exponent = d.as_tuple()
  494. value = reduce(lambda left, right: left * 10 + right, guts)
  495. if sign:
  496. value = -value
  497. return [b"decimal", value, exponent]
  498. def unpersistable(self, reason, sxp=None):
  499. """
  500. (internal) Returns an sexp: (unpersistable "reason"). Utility method
  501. for making note that a particular object could not be serialized.
  502. """
  503. if sxp is None:
  504. sxp = []
  505. sxp.append(unpersistable_atom)
  506. if isinstance(reason, str):
  507. reason = reason.encode("utf-8")
  508. sxp.append(reason)
  509. return sxp
  510. class _Unjellier:
  511. def __init__(self, taster, persistentLoad, invoker):
  512. self.taster = taster
  513. self.persistentLoad = persistentLoad
  514. self.references = {}
  515. self.postCallbacks = []
  516. self.invoker = invoker
  517. def unjellyFull(self, obj):
  518. o = self.unjelly(obj)
  519. for m in self.postCallbacks:
  520. m()
  521. return o
  522. def _maybePostUnjelly(self, unjellied):
  523. """
  524. If the given object has support for the C{postUnjelly} hook, set it up
  525. to be called at the end of deserialization.
  526. @param unjellied: an object that has already been unjellied.
  527. @return: C{unjellied}
  528. """
  529. if hasattr(unjellied, "postUnjelly"):
  530. self.postCallbacks.append(unjellied.postUnjelly)
  531. return unjellied
  532. def unjelly(self, obj):
  533. if type(obj) is not list:
  534. return obj
  535. jelTypeBytes = obj[0]
  536. if not self.taster.isTypeAllowed(jelTypeBytes):
  537. raise InsecureJelly(jelTypeBytes)
  538. regClass = unjellyableRegistry.get(jelTypeBytes)
  539. if regClass is not None:
  540. method = getattr(_createBlank(regClass), "unjellyFor", regClass)
  541. return self._maybePostUnjelly(method(self, obj))
  542. regFactory = unjellyableFactoryRegistry.get(jelTypeBytes)
  543. if regFactory is not None:
  544. return self._maybePostUnjelly(regFactory(self.unjelly(obj[1])))
  545. jelTypeText = nativeString(jelTypeBytes)
  546. thunk = getattr(self, "_unjelly_%s" % jelTypeText, None)
  547. if thunk is not None:
  548. return thunk(obj[1:])
  549. else:
  550. nameSplit = jelTypeText.split(".")
  551. modName = ".".join(nameSplit[:-1])
  552. if not self.taster.isModuleAllowed(modName):
  553. raise InsecureJelly(
  554. f"Module {modName} not allowed (in type {jelTypeText})."
  555. )
  556. clz = namedObject(jelTypeText)
  557. if not self.taster.isClassAllowed(clz):
  558. raise InsecureJelly("Class %s not allowed." % jelTypeText)
  559. return self._genericUnjelly(clz, obj[1])
  560. def _genericUnjelly(self, cls, state):
  561. """
  562. Unjelly a type for which no specific unjellier is registered, but which
  563. is nonetheless allowed.
  564. @param cls: the class of the instance we are unjellying.
  565. @type cls: L{type}
  566. @param state: The jellied representation of the object's state; its
  567. C{__dict__} unless it has a C{__setstate__} that takes something
  568. else.
  569. @type state: L{list}
  570. @return: the new, unjellied instance.
  571. """
  572. return self._maybePostUnjelly(_newInstance(cls, self.unjelly(state)))
  573. def _unjelly_None(self, exp):
  574. return None
  575. def _unjelly_unicode(self, exp):
  576. return str(exp[0], "UTF-8")
  577. def _unjelly_decimal(self, exp):
  578. """
  579. Unjelly decimal objects.
  580. """
  581. value = exp[0]
  582. exponent = exp[1]
  583. if value < 0:
  584. sign = 1
  585. else:
  586. sign = 0
  587. guts = decimal.Decimal(value).as_tuple()[1]
  588. return decimal.Decimal((sign, guts, exponent))
  589. def _unjelly_boolean(self, exp):
  590. assert exp[0] in (b"true", b"false")
  591. return exp[0] == b"true"
  592. def _unjelly_datetime(self, exp):
  593. return datetime.datetime(*map(int, exp[0].split()))
  594. def _unjelly_date(self, exp):
  595. return datetime.date(*map(int, exp[0].split()))
  596. def _unjelly_time(self, exp):
  597. return datetime.time(*map(int, exp[0].split()))
  598. def _unjelly_timedelta(self, exp):
  599. days, seconds, microseconds = map(int, exp[0].split())
  600. return datetime.timedelta(days=days, seconds=seconds, microseconds=microseconds)
  601. def unjellyInto(self, obj, loc, jel):
  602. o = self.unjelly(jel)
  603. if isinstance(o, NotKnown):
  604. o.addDependant(obj, loc)
  605. obj[loc] = o
  606. return o
  607. def _unjelly_dereference(self, lst):
  608. refid = lst[0]
  609. x = self.references.get(refid)
  610. if x is not None:
  611. return x
  612. der = _Dereference(refid)
  613. self.references[refid] = der
  614. return der
  615. def _unjelly_reference(self, lst):
  616. refid = lst[0]
  617. exp = lst[1]
  618. o = self.unjelly(exp)
  619. ref = self.references.get(refid)
  620. if ref is None:
  621. self.references[refid] = o
  622. elif isinstance(ref, NotKnown):
  623. ref.resolveDependants(o)
  624. self.references[refid] = o
  625. else:
  626. assert 0, "Multiple references with same ID!"
  627. return o
  628. def _unjelly_tuple(self, lst):
  629. l = list(range(len(lst)))
  630. finished = 1
  631. for elem in l:
  632. if isinstance(self.unjellyInto(l, elem, lst[elem]), NotKnown):
  633. finished = 0
  634. if finished:
  635. return tuple(l)
  636. else:
  637. return _Tuple(l)
  638. def _unjelly_list(self, lst):
  639. l = list(range(len(lst)))
  640. for elem in l:
  641. self.unjellyInto(l, elem, lst[elem])
  642. return l
  643. def _unjellySetOrFrozenset(self, lst, containerType):
  644. """
  645. Helper method to unjelly set or frozenset.
  646. @param lst: the content of the set.
  647. @type lst: C{list}
  648. @param containerType: the type of C{set} to use.
  649. """
  650. l = list(range(len(lst)))
  651. finished = True
  652. for elem in l:
  653. data = self.unjellyInto(l, elem, lst[elem])
  654. if isinstance(data, NotKnown):
  655. finished = False
  656. if not finished:
  657. return _Container(l, containerType)
  658. else:
  659. return containerType(l)
  660. def _unjelly_set(self, lst):
  661. """
  662. Unjelly set using the C{set} builtin.
  663. """
  664. return self._unjellySetOrFrozenset(lst, set)
  665. def _unjelly_frozenset(self, lst):
  666. """
  667. Unjelly frozenset using the C{frozenset} builtin.
  668. """
  669. return self._unjellySetOrFrozenset(lst, frozenset)
  670. def _unjelly_dictionary(self, lst):
  671. d = {}
  672. for k, v in lst:
  673. kvd = _DictKeyAndValue(d)
  674. self.unjellyInto(kvd, 0, k)
  675. self.unjellyInto(kvd, 1, v)
  676. return d
  677. def _unjelly_module(self, rest):
  678. moduleName = nativeString(rest[0])
  679. if type(moduleName) != str:
  680. raise InsecureJelly("Attempted to unjelly a module with a non-string name.")
  681. if not self.taster.isModuleAllowed(moduleName):
  682. raise InsecureJelly(f"Attempted to unjelly module named {moduleName!r}")
  683. mod = __import__(moduleName, {}, {}, "x")
  684. return mod
  685. def _unjelly_class(self, rest):
  686. cname = nativeString(rest[0])
  687. clist = cname.split(nativeString("."))
  688. modName = nativeString(".").join(clist[:-1])
  689. if not self.taster.isModuleAllowed(modName):
  690. raise InsecureJelly("module %s not allowed" % modName)
  691. klaus = namedObject(cname)
  692. objType = type(klaus)
  693. if objType is not type:
  694. raise InsecureJelly(
  695. "class %r unjellied to something that isn't a class: %r"
  696. % (cname, klaus)
  697. )
  698. if not self.taster.isClassAllowed(klaus):
  699. raise InsecureJelly("class not allowed: %s" % qual(klaus))
  700. return klaus
  701. def _unjelly_function(self, rest):
  702. fname = nativeString(rest[0])
  703. modSplit = fname.split(nativeString("."))
  704. modName = nativeString(".").join(modSplit[:-1])
  705. if not self.taster.isModuleAllowed(modName):
  706. raise InsecureJelly("Module not allowed: %s" % modName)
  707. # XXX do I need an isFunctionAllowed?
  708. function = namedAny(fname)
  709. return function
  710. def _unjelly_persistent(self, rest):
  711. if self.persistentLoad:
  712. pload = self.persistentLoad(rest[0], self)
  713. return pload
  714. else:
  715. return Unpersistable("Persistent callback not found")
  716. def _unjelly_instance(self, rest):
  717. """
  718. (internal) Unjelly an instance.
  719. Called to handle the deprecated I{instance} token.
  720. @param rest: The s-expression representing the instance.
  721. @return: The unjellied instance.
  722. """
  723. warnings.warn_explicit(
  724. "Unjelly support for the instance atom is deprecated since "
  725. "Twisted 15.0.0. Upgrade peer for modern instance support.",
  726. category=DeprecationWarning,
  727. filename="",
  728. lineno=0,
  729. )
  730. clz = self.unjelly(rest[0])
  731. return self._genericUnjelly(clz, rest[1])
  732. def _unjelly_unpersistable(self, rest):
  733. return Unpersistable(f"Unpersistable data: {rest[0]}")
  734. def _unjelly_method(self, rest):
  735. """
  736. (internal) Unjelly a method.
  737. """
  738. im_name = rest[0]
  739. im_self = self.unjelly(rest[1])
  740. im_class = self.unjelly(rest[2])
  741. if not isinstance(im_class, type):
  742. raise InsecureJelly("Method found with non-class class.")
  743. if im_name in im_class.__dict__:
  744. if im_self is None:
  745. im = getattr(im_class, im_name)
  746. elif isinstance(im_self, NotKnown):
  747. im = _InstanceMethod(im_name, im_self, im_class)
  748. else:
  749. im = types.MethodType(
  750. im_class.__dict__[im_name], im_self, *([im_class] * (False))
  751. )
  752. else:
  753. raise TypeError("instance method changed")
  754. return im
  755. #### Published Interface.
  756. class InsecureJelly(Exception):
  757. """
  758. This exception will be raised when a jelly is deemed `insecure'; e.g. it
  759. contains a type, class, or module disallowed by the specified `taster'
  760. """
  761. class DummySecurityOptions:
  762. """
  763. DummySecurityOptions() -> insecure security options
  764. Dummy security options -- this class will allow anything.
  765. """
  766. def isModuleAllowed(self, moduleName):
  767. """
  768. DummySecurityOptions.isModuleAllowed(moduleName) -> boolean
  769. returns 1 if a module by that name is allowed, 0 otherwise
  770. """
  771. return 1
  772. def isClassAllowed(self, klass):
  773. """
  774. DummySecurityOptions.isClassAllowed(class) -> boolean
  775. Assumes the module has already been allowed. Returns 1 if the given
  776. class is allowed, 0 otherwise.
  777. """
  778. return 1
  779. def isTypeAllowed(self, typeName):
  780. """
  781. DummySecurityOptions.isTypeAllowed(typeName) -> boolean
  782. Returns 1 if the given type is allowed, 0 otherwise.
  783. """
  784. return 1
  785. class SecurityOptions:
  786. """
  787. This will by default disallow everything, except for 'none'.
  788. """
  789. basicTypes = [
  790. "dictionary",
  791. "list",
  792. "tuple",
  793. "reference",
  794. "dereference",
  795. "unpersistable",
  796. "persistent",
  797. "long_int",
  798. "long",
  799. "dict",
  800. ]
  801. def __init__(self):
  802. """
  803. SecurityOptions() initialize.
  804. """
  805. # I don't believe any of these types can ever pose a security hazard,
  806. # except perhaps "reference"...
  807. self.allowedTypes = {
  808. b"None": 1,
  809. b"bool": 1,
  810. b"boolean": 1,
  811. b"string": 1,
  812. b"str": 1,
  813. b"int": 1,
  814. b"float": 1,
  815. b"datetime": 1,
  816. b"time": 1,
  817. b"date": 1,
  818. b"timedelta": 1,
  819. b"NoneType": 1,
  820. b"unicode": 1,
  821. b"decimal": 1,
  822. b"set": 1,
  823. b"frozenset": 1,
  824. }
  825. self.allowedModules = {}
  826. self.allowedClasses = {}
  827. def allowBasicTypes(self):
  828. """
  829. Allow all `basic' types. (Dictionary and list. Int, string, and float
  830. are implicitly allowed.)
  831. """
  832. self.allowTypes(*self.basicTypes)
  833. def allowTypes(self, *types):
  834. """
  835. SecurityOptions.allowTypes(typeString): Allow a particular type, by its
  836. name.
  837. """
  838. for typ in types:
  839. if isinstance(typ, str):
  840. typ = typ.encode("utf-8")
  841. if not isinstance(typ, bytes):
  842. typ = qual(typ)
  843. self.allowedTypes[typ] = 1
  844. def allowInstancesOf(self, *classes):
  845. """
  846. SecurityOptions.allowInstances(klass, klass, ...): allow instances
  847. of the specified classes
  848. This will also allow the 'instance', 'class' (renamed 'classobj' in
  849. Python 2.3), and 'module' types, as well as basic types.
  850. """
  851. self.allowBasicTypes()
  852. self.allowTypes("instance", "class", "classobj", "module")
  853. for klass in classes:
  854. self.allowTypes(qual(klass))
  855. self.allowModules(klass.__module__)
  856. self.allowedClasses[klass] = 1
  857. def allowModules(self, *modules):
  858. """
  859. SecurityOptions.allowModules(module, module, ...): allow modules by
  860. name. This will also allow the 'module' type.
  861. """
  862. for module in modules:
  863. if type(module) == types.ModuleType:
  864. module = module.__name__
  865. if not isinstance(module, bytes):
  866. module = module.encode("utf-8")
  867. self.allowedModules[module] = 1
  868. def isModuleAllowed(self, moduleName):
  869. """
  870. SecurityOptions.isModuleAllowed(moduleName) -> boolean
  871. returns 1 if a module by that name is allowed, 0 otherwise
  872. """
  873. if not isinstance(moduleName, bytes):
  874. moduleName = moduleName.encode("utf-8")
  875. return moduleName in self.allowedModules
  876. def isClassAllowed(self, klass):
  877. """
  878. SecurityOptions.isClassAllowed(class) -> boolean
  879. Assumes the module has already been allowed. Returns 1 if the given
  880. class is allowed, 0 otherwise.
  881. """
  882. return klass in self.allowedClasses
  883. def isTypeAllowed(self, typeName):
  884. """
  885. SecurityOptions.isTypeAllowed(typeName) -> boolean
  886. Returns 1 if the given type is allowed, 0 otherwise.
  887. """
  888. if not isinstance(typeName, bytes):
  889. typeName = typeName.encode("utf-8")
  890. return typeName in self.allowedTypes or b"." in typeName
  891. globalSecurity = SecurityOptions()
  892. globalSecurity.allowBasicTypes()
  893. def jelly(object, taster=DummySecurityOptions(), persistentStore=None, invoker=None):
  894. """
  895. Serialize to s-expression.
  896. Returns a list which is the serialized representation of an object. An
  897. optional 'taster' argument takes a SecurityOptions and will mark any
  898. insecure objects as unpersistable rather than serializing them.
  899. """
  900. return _Jellier(taster, persistentStore, invoker).jelly(object)
  901. def unjelly(sexp, taster=DummySecurityOptions(), persistentLoad=None, invoker=None):
  902. """
  903. Unserialize from s-expression.
  904. Takes a list that was the result from a call to jelly() and unserializes
  905. an arbitrary object from it. The optional 'taster' argument, an instance
  906. of SecurityOptions, will cause an InsecureJelly exception to be raised if a
  907. disallowed type, module, or class attempted to unserialize.
  908. """
  909. return _Unjellier(taster, persistentLoad, invoker).unjellyFull(sexp)