Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

aot.py 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. # -*- test-case-name: twisted.test.test_persisted -*-
  2. # Copyright (c) Twisted Matrix Laboratories.
  3. # See LICENSE for details.
  4. """
  5. AOT: Abstract Object Trees
  6. The source-code-marshallin'est abstract-object-serializin'est persister
  7. this side of Marmalade!
  8. """
  9. import copyreg as copy_reg
  10. import re
  11. import types
  12. from tokenize import generate_tokens as tokenize
  13. from twisted.persisted import crefutil
  14. from twisted.python import log, reflect
  15. from twisted.python.compat import _constructMethod
  16. ###########################
  17. # Abstract Object Classes #
  18. ###########################
  19. # "\0" in a getSource means "insert variable-width indention here".
  20. # see `indentify'.
  21. class Named:
  22. def __init__(self, name):
  23. self.name = name
  24. class Class(Named):
  25. def getSource(self):
  26. return "Class(%r)" % self.name
  27. class Function(Named):
  28. def getSource(self):
  29. return "Function(%r)" % self.name
  30. class Module(Named):
  31. def getSource(self):
  32. return "Module(%r)" % self.name
  33. class InstanceMethod:
  34. def __init__(self, name, klass, inst):
  35. if not (
  36. isinstance(inst, Ref)
  37. or isinstance(inst, Instance)
  38. or isinstance(inst, Deref)
  39. ):
  40. raise TypeError("%s isn't an Instance, Ref, or Deref!" % inst)
  41. self.name = name
  42. self.klass = klass
  43. self.instance = inst
  44. def getSource(self):
  45. return "InstanceMethod({!r}, {!r}, \n\0{})".format(
  46. self.name,
  47. self.klass,
  48. prettify(self.instance),
  49. )
  50. class _NoStateObj:
  51. pass
  52. NoStateObj = _NoStateObj()
  53. _SIMPLE_BUILTINS = [
  54. bool,
  55. bytes,
  56. str,
  57. int,
  58. float,
  59. complex,
  60. type(None),
  61. slice,
  62. type(Ellipsis),
  63. ]
  64. class Instance:
  65. def __init__(self, className, __stateObj__=NoStateObj, **state):
  66. if not isinstance(className, str):
  67. raise TypeError("%s isn't a string!" % className)
  68. self.klass = className
  69. if __stateObj__ is not NoStateObj:
  70. self.state = __stateObj__
  71. self.stateIsDict = 0
  72. else:
  73. self.state = state
  74. self.stateIsDict = 1
  75. def getSource(self):
  76. # XXX make state be foo=bar instead of a dict.
  77. if self.stateIsDict:
  78. stateDict = self.state
  79. elif isinstance(self.state, Ref) and isinstance(self.state.obj, dict):
  80. stateDict = self.state.obj
  81. else:
  82. stateDict = None
  83. if stateDict is not None:
  84. try:
  85. return f"Instance({self.klass!r}, {dictToKW(stateDict)})"
  86. except NonFormattableDict:
  87. return f"Instance({self.klass!r}, {prettify(stateDict)})"
  88. return f"Instance({self.klass!r}, {prettify(self.state)})"
  89. class Ref:
  90. def __init__(self, *args):
  91. # blargh, lame.
  92. if len(args) == 2:
  93. self.refnum = args[0]
  94. self.obj = args[1]
  95. elif not args:
  96. self.refnum = None
  97. self.obj = None
  98. def setRef(self, num):
  99. if self.refnum:
  100. raise ValueError(f"Error setting id {num}, I already have {self.refnum}")
  101. self.refnum = num
  102. def setObj(self, obj):
  103. if self.obj:
  104. raise ValueError(f"Error setting obj {obj}, I already have {self.obj}")
  105. self.obj = obj
  106. def getSource(self):
  107. if self.obj is None:
  108. raise RuntimeError(
  109. "Don't try to display me before setting an object on me!"
  110. )
  111. if self.refnum:
  112. return "Ref(%d, \n\0%s)" % (self.refnum, prettify(self.obj))
  113. return prettify(self.obj)
  114. class Deref:
  115. def __init__(self, num):
  116. self.refnum = num
  117. def getSource(self):
  118. return "Deref(%d)" % self.refnum
  119. __repr__ = getSource
  120. class Copyreg:
  121. def __init__(self, loadfunc, state):
  122. self.loadfunc = loadfunc
  123. self.state = state
  124. def getSource(self):
  125. return f"Copyreg({self.loadfunc!r}, {prettify(self.state)})"
  126. ###############
  127. # Marshalling #
  128. ###############
  129. def getSource(ao):
  130. """Pass me an AO, I'll return a nicely-formatted source representation."""
  131. return indentify("app = " + prettify(ao))
  132. class NonFormattableDict(Exception):
  133. """A dictionary was not formattable."""
  134. r = re.compile("[a-zA-Z_][a-zA-Z0-9_]*$")
  135. def dictToKW(d):
  136. out = []
  137. items = list(d.items())
  138. items.sort()
  139. for k, v in items:
  140. if not isinstance(k, str):
  141. raise NonFormattableDict("%r ain't a string" % k)
  142. if not r.match(k):
  143. raise NonFormattableDict("%r ain't an identifier" % k)
  144. out.append(f"\n\0{k}={prettify(v)},")
  145. return "".join(out)
  146. def prettify(obj):
  147. if hasattr(obj, "getSource"):
  148. return obj.getSource()
  149. else:
  150. # basic type
  151. t = type(obj)
  152. if t in _SIMPLE_BUILTINS:
  153. return repr(obj)
  154. elif t is dict:
  155. out = ["{"]
  156. for k, v in obj.items():
  157. out.append(f"\n\0{prettify(k)}: {prettify(v)},")
  158. out.append(len(obj) and "\n\0}" or "}")
  159. return "".join(out)
  160. elif t is list:
  161. out = ["["]
  162. for x in obj:
  163. out.append("\n\0%s," % prettify(x))
  164. out.append(len(obj) and "\n\0]" or "]")
  165. return "".join(out)
  166. elif t is tuple:
  167. out = ["("]
  168. for x in obj:
  169. out.append("\n\0%s," % prettify(x))
  170. out.append(len(obj) and "\n\0)" or ")")
  171. return "".join(out)
  172. else:
  173. raise TypeError(f"Unsupported type {t} when trying to prettify {obj}.")
  174. def indentify(s):
  175. out = []
  176. stack = []
  177. l = ["", s]
  178. for (
  179. tokenType,
  180. tokenString,
  181. (startRow, startColumn),
  182. (endRow, endColumn),
  183. logicalLine,
  184. ) in tokenize(l.pop):
  185. if tokenString in ["[", "(", "{"]:
  186. stack.append(tokenString)
  187. elif tokenString in ["]", ")", "}"]:
  188. stack.pop()
  189. if tokenString == "\0":
  190. out.append(" " * len(stack))
  191. else:
  192. out.append(tokenString)
  193. return "".join(out)
  194. ###########
  195. # Unjelly #
  196. ###########
  197. def unjellyFromAOT(aot):
  198. """
  199. Pass me an Abstract Object Tree, and I'll unjelly it for you.
  200. """
  201. return AOTUnjellier().unjelly(aot)
  202. def unjellyFromSource(stringOrFile):
  203. """
  204. Pass me a string of code or a filename that defines an 'app' variable (in
  205. terms of Abstract Objects!), and I'll execute it and unjelly the resulting
  206. AOT for you, returning a newly unpersisted Application object!
  207. """
  208. ns = {
  209. "Instance": Instance,
  210. "InstanceMethod": InstanceMethod,
  211. "Class": Class,
  212. "Function": Function,
  213. "Module": Module,
  214. "Ref": Ref,
  215. "Deref": Deref,
  216. "Copyreg": Copyreg,
  217. }
  218. if hasattr(stringOrFile, "read"):
  219. source = stringOrFile.read()
  220. else:
  221. source = stringOrFile
  222. code = compile(source, "<source>", "exec")
  223. eval(code, ns, ns)
  224. if "app" in ns:
  225. return unjellyFromAOT(ns["app"])
  226. else:
  227. raise ValueError("%s needs to define an 'app', it didn't!" % stringOrFile)
  228. class AOTUnjellier:
  229. """I handle the unjellying of an Abstract Object Tree.
  230. See AOTUnjellier.unjellyAO
  231. """
  232. def __init__(self):
  233. self.references = {}
  234. self.stack = []
  235. self.afterUnjelly = []
  236. ##
  237. # unjelly helpers (copied pretty much directly from (now deleted) marmalade)
  238. ##
  239. def unjellyLater(self, node):
  240. """Unjelly a node, later."""
  241. d = crefutil._Defer()
  242. self.unjellyInto(d, 0, node)
  243. return d
  244. def unjellyInto(self, obj, loc, ao):
  245. """Utility method for unjellying one object into another.
  246. This automates the handling of backreferences.
  247. """
  248. o = self.unjellyAO(ao)
  249. obj[loc] = o
  250. if isinstance(o, crefutil.NotKnown):
  251. o.addDependant(obj, loc)
  252. return o
  253. def callAfter(self, callable, result):
  254. if isinstance(result, crefutil.NotKnown):
  255. listResult = [None]
  256. result.addDependant(listResult, 1)
  257. else:
  258. listResult = [result]
  259. self.afterUnjelly.append((callable, listResult))
  260. def unjellyAttribute(self, instance, attrName, ao):
  261. # XXX this is unused????
  262. """Utility method for unjellying into instances of attributes.
  263. Use this rather than unjellyAO unless you like surprising bugs!
  264. Alternatively, you can use unjellyInto on your instance's __dict__.
  265. """
  266. self.unjellyInto(instance.__dict__, attrName, ao)
  267. def unjellyAO(self, ao):
  268. """Unjelly an Abstract Object and everything it contains.
  269. I return the real object.
  270. """
  271. self.stack.append(ao)
  272. t = type(ao)
  273. if t in _SIMPLE_BUILTINS:
  274. return ao
  275. elif t is list:
  276. l = []
  277. for x in ao:
  278. l.append(None)
  279. self.unjellyInto(l, len(l) - 1, x)
  280. return l
  281. elif t is tuple:
  282. l = []
  283. tuple_ = tuple
  284. for x in ao:
  285. l.append(None)
  286. if isinstance(self.unjellyInto(l, len(l) - 1, x), crefutil.NotKnown):
  287. tuple_ = crefutil._Tuple
  288. return tuple_(l)
  289. elif t is dict:
  290. d = {}
  291. for k, v in ao.items():
  292. kvd = crefutil._DictKeyAndValue(d)
  293. self.unjellyInto(kvd, 0, k)
  294. self.unjellyInto(kvd, 1, v)
  295. return d
  296. else:
  297. # Abstract Objects
  298. c = ao.__class__
  299. if c is Module:
  300. return reflect.namedModule(ao.name)
  301. elif c in [Class, Function] or issubclass(c, type):
  302. return reflect.namedObject(ao.name)
  303. elif c is InstanceMethod:
  304. im_name = ao.name
  305. im_class = reflect.namedObject(ao.klass)
  306. im_self = self.unjellyAO(ao.instance)
  307. if im_name in im_class.__dict__:
  308. if im_self is None:
  309. return getattr(im_class, im_name)
  310. elif isinstance(im_self, crefutil.NotKnown):
  311. return crefutil._InstanceMethod(im_name, im_self, im_class)
  312. else:
  313. return _constructMethod(im_class, im_name, im_self)
  314. else:
  315. raise TypeError("instance method changed")
  316. elif c is Instance:
  317. klass = reflect.namedObject(ao.klass)
  318. state = self.unjellyAO(ao.state)
  319. inst = klass.__new__(klass)
  320. if hasattr(klass, "__setstate__"):
  321. self.callAfter(inst.__setstate__, state)
  322. else:
  323. inst.__dict__ = state
  324. return inst
  325. elif c is Ref:
  326. o = self.unjellyAO(ao.obj) # THIS IS CHANGING THE REF OMG
  327. refkey = ao.refnum
  328. ref = self.references.get(refkey)
  329. if ref is None:
  330. self.references[refkey] = o
  331. elif isinstance(ref, crefutil.NotKnown):
  332. ref.resolveDependants(o)
  333. self.references[refkey] = o
  334. elif refkey is None:
  335. # This happens when you're unjellying from an AOT not read from source
  336. pass
  337. else:
  338. raise ValueError(
  339. "Multiple references with the same ID: %s, %s, %s!"
  340. % (ref, refkey, ao)
  341. )
  342. return o
  343. elif c is Deref:
  344. num = ao.refnum
  345. ref = self.references.get(num)
  346. if ref is None:
  347. der = crefutil._Dereference(num)
  348. self.references[num] = der
  349. return der
  350. return ref
  351. elif c is Copyreg:
  352. loadfunc = reflect.namedObject(ao.loadfunc)
  353. d = self.unjellyLater(ao.state).addCallback(
  354. lambda result, _l: _l(*result), loadfunc
  355. )
  356. return d
  357. else:
  358. raise TypeError("Unsupported AOT type: %s" % t)
  359. def unjelly(self, ao):
  360. try:
  361. l = [None]
  362. self.unjellyInto(l, 0, ao)
  363. for func, v in self.afterUnjelly:
  364. func(v[0])
  365. return l[0]
  366. except BaseException:
  367. log.msg("Error jellying object! Stacktrace follows::")
  368. log.msg("\n".join(map(repr, self.stack)))
  369. raise
  370. #########
  371. # Jelly #
  372. #########
  373. def jellyToAOT(obj):
  374. """Convert an object to an Abstract Object Tree."""
  375. return AOTJellier().jelly(obj)
  376. def jellyToSource(obj, file=None):
  377. """
  378. Pass me an object and, optionally, a file object.
  379. I'll convert the object to an AOT either return it (if no file was
  380. specified) or write it to the file.
  381. """
  382. aot = jellyToAOT(obj)
  383. if file:
  384. file.write(getSource(aot).encode("utf-8"))
  385. else:
  386. return getSource(aot)
  387. def _classOfMethod(methodObject):
  388. """
  389. Get the associated class of the given method object.
  390. @param methodObject: a bound method
  391. @type methodObject: L{types.MethodType}
  392. @return: a class
  393. @rtype: L{type}
  394. """
  395. return methodObject.__self__.__class__
  396. def _funcOfMethod(methodObject):
  397. """
  398. Get the associated function of the given method object.
  399. @param methodObject: a bound method
  400. @type methodObject: L{types.MethodType}
  401. @return: the function implementing C{methodObject}
  402. @rtype: L{types.FunctionType}
  403. """
  404. return methodObject.__func__
  405. def _selfOfMethod(methodObject):
  406. """
  407. Get the object that a bound method is bound to.
  408. @param methodObject: a bound method
  409. @type methodObject: L{types.MethodType}
  410. @return: the C{self} passed to C{methodObject}
  411. @rtype: L{object}
  412. """
  413. return methodObject.__self__
  414. class AOTJellier:
  415. def __init__(self):
  416. # dict of {id(obj): (obj, node)}
  417. self.prepared = {}
  418. self._ref_id = 0
  419. self.stack = []
  420. def prepareForRef(self, aoref, object):
  421. """I prepare an object for later referencing, by storing its id()
  422. and its _AORef in a cache."""
  423. self.prepared[id(object)] = aoref
  424. def jellyToAO(self, obj):
  425. """I turn an object into an AOT and return it."""
  426. objType = type(obj)
  427. self.stack.append(repr(obj))
  428. # immutable: We don't care if these have multiple refs!
  429. if objType in _SIMPLE_BUILTINS:
  430. retval = obj
  431. elif issubclass(objType, types.MethodType):
  432. # TODO: make methods 'prefer' not to jelly the object internally,
  433. # so that the object will show up where it's referenced first NOT
  434. # by a method.
  435. retval = InstanceMethod(
  436. _funcOfMethod(obj).__name__,
  437. reflect.qual(_classOfMethod(obj)),
  438. self.jellyToAO(_selfOfMethod(obj)),
  439. )
  440. elif issubclass(objType, types.ModuleType):
  441. retval = Module(obj.__name__)
  442. elif issubclass(objType, type):
  443. retval = Class(reflect.qual(obj))
  444. elif objType is types.FunctionType:
  445. retval = Function(reflect.fullFuncName(obj))
  446. else: # mutable! gotta watch for refs.
  447. # Marmalade had the nicety of being able to just stick a 'reference' attribute
  448. # on any Node object that was referenced, but in AOT, the referenced object
  449. # is *inside* of a Ref call (Ref(num, obj) instead of
  450. # <objtype ... reference="1">). The problem is, especially for built-in types,
  451. # I can't just assign some attribute to them to give them a refnum. So, I have
  452. # to "wrap" a Ref(..) around them later -- that's why I put *everything* that's
  453. # mutable inside one. The Ref() class will only print the "Ref(..)" around an
  454. # object if it has a Reference explicitly attached.
  455. if id(obj) in self.prepared:
  456. oldRef = self.prepared[id(obj)]
  457. if oldRef.refnum:
  458. # it's been referenced already
  459. key = oldRef.refnum
  460. else:
  461. # it hasn't been referenced yet
  462. self._ref_id = self._ref_id + 1
  463. key = self._ref_id
  464. oldRef.setRef(key)
  465. return Deref(key)
  466. retval = Ref()
  467. def _stateFrom(state):
  468. retval.setObj(
  469. Instance(reflect.qual(obj.__class__), self.jellyToAO(state))
  470. )
  471. self.prepareForRef(retval, obj)
  472. if objType is list:
  473. retval.setObj([self.jellyToAO(o) for o in obj]) # hah!
  474. elif objType is tuple:
  475. retval.setObj(tuple(map(self.jellyToAO, obj)))
  476. elif objType is dict:
  477. d = {}
  478. for k, v in obj.items():
  479. d[self.jellyToAO(k)] = self.jellyToAO(v)
  480. retval.setObj(d)
  481. elif objType in copy_reg.dispatch_table:
  482. unpickleFunc, state = copy_reg.dispatch_table[objType](obj)
  483. retval.setObj(
  484. Copyreg(reflect.fullFuncName(unpickleFunc), self.jellyToAO(state))
  485. )
  486. elif hasattr(obj, "__getstate__"):
  487. _stateFrom(obj.__getstate__())
  488. elif hasattr(obj, "__dict__"):
  489. _stateFrom(obj.__dict__)
  490. else:
  491. raise TypeError("Unsupported type: %s" % objType.__name__)
  492. del self.stack[-1]
  493. return retval
  494. def jelly(self, obj):
  495. try:
  496. ao = self.jellyToAO(obj)
  497. return ao
  498. except BaseException:
  499. log.msg("Error jellying object! Stacktrace follows::")
  500. log.msg("\n".join(self.stack))
  501. raise