Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

reporter.py 40KB

1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278
  1. # -*- test-case-name: twisted.trial.test.test_reporter -*-
  2. # Copyright (c) Twisted Matrix Laboratories.
  3. # See LICENSE for details.
  4. #
  5. # Maintainer: Jonathan Lange
  6. """
  7. Defines classes that handle the results of tests.
  8. """
  9. import os
  10. import sys
  11. import time
  12. import unittest as pyunit
  13. import warnings
  14. from collections import OrderedDict
  15. from types import TracebackType
  16. from typing import TYPE_CHECKING, List, Tuple, Type, Union
  17. from zope.interface import implementer
  18. from typing_extensions import TypeAlias
  19. from twisted.python import log, reflect
  20. from twisted.python.components import proxyForInterface
  21. from twisted.python.failure import Failure
  22. from twisted.python.util import untilConcludes
  23. from twisted.trial import itrial, util
  24. if TYPE_CHECKING:
  25. from ._synctest import Todo
  26. try:
  27. from subunit import TestProtocolClient # type: ignore[import]
  28. except ImportError:
  29. TestProtocolClient = None
  30. ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType]
  31. XUnitFailure = Union[ExcInfo, Tuple[None, None, None]]
  32. TrialFailure = Union[XUnitFailure, Failure]
  33. def _makeTodo(value: str) -> "Todo":
  34. """
  35. Return a L{Todo} object built from C{value}.
  36. This is a synonym for L{twisted.trial.unittest.makeTodo}, but imported
  37. locally to avoid circular imports.
  38. @param value: A string or a tuple of C{(errors, reason)}, where C{errors}
  39. is either a single exception class or an iterable of exception classes.
  40. @return: A L{Todo} object.
  41. """
  42. from twisted.trial.unittest import makeTodo
  43. return makeTodo(value)
  44. class BrokenTestCaseWarning(Warning):
  45. """
  46. Emitted as a warning when an exception occurs in one of setUp or tearDown.
  47. """
  48. class SafeStream:
  49. """
  50. Wraps a stream object so that all C{write} calls are wrapped in
  51. L{untilConcludes<twisted.python.util.untilConcludes>}.
  52. """
  53. def __init__(self, original):
  54. self.original = original
  55. def __getattr__(self, name):
  56. return getattr(self.original, name)
  57. def write(self, *a, **kw):
  58. return untilConcludes(self.original.write, *a, **kw)
  59. @implementer(itrial.IReporter)
  60. class TestResult(pyunit.TestResult):
  61. """
  62. Accumulates the results of several L{twisted.trial.unittest.TestCase}s.
  63. @ivar successes: count the number of successes achieved by the test run.
  64. @type successes: C{int}
  65. """
  66. # Used when no todo provided to addExpectedFailure or addUnexpectedSuccess.
  67. _DEFAULT_TODO = "Test expected to fail"
  68. skips: List[Tuple[itrial.ITestCase, str]]
  69. expectedFailures: List[Tuple[itrial.ITestCase, str, "Todo"]] # type: ignore[assignment]
  70. unexpectedSuccesses: List[Tuple[itrial.ITestCase, str]] # type: ignore[assignment]
  71. successes: int
  72. def __init__(self):
  73. super().__init__()
  74. self.skips = []
  75. self.expectedFailures = []
  76. self.unexpectedSuccesses = []
  77. self.successes = 0
  78. self._timings = []
  79. def __repr__(self) -> str:
  80. return "<%s run=%d errors=%d failures=%d todos=%d dones=%d skips=%d>" % (
  81. reflect.qual(self.__class__),
  82. self.testsRun,
  83. len(self.errors),
  84. len(self.failures),
  85. len(self.expectedFailures),
  86. len(self.skips),
  87. len(self.unexpectedSuccesses),
  88. )
  89. def _getTime(self):
  90. return time.time()
  91. def _getFailure(self, error):
  92. """
  93. Convert a C{sys.exc_info()}-style tuple to a L{Failure}, if necessary.
  94. """
  95. is_exc_info_tuple = isinstance(error, tuple) and len(error) == 3
  96. if is_exc_info_tuple:
  97. return Failure(error[1], error[0], error[2])
  98. elif isinstance(error, Failure):
  99. return error
  100. raise TypeError(f"Cannot convert {error} to a Failure")
  101. def startTest(self, test):
  102. """
  103. This must be called before the given test is commenced.
  104. @type test: L{pyunit.TestCase}
  105. """
  106. super().startTest(test)
  107. self._testStarted = self._getTime()
  108. def stopTest(self, test):
  109. """
  110. This must be called after the given test is completed.
  111. @type test: L{pyunit.TestCase}
  112. """
  113. super().stopTest(test)
  114. self._lastTime = self._getTime() - self._testStarted
  115. def addFailure(self, test, fail):
  116. """
  117. Report a failed assertion for the given test.
  118. @type test: L{pyunit.TestCase}
  119. @type fail: L{Failure} or L{tuple}
  120. """
  121. self.failures.append((test, self._getFailure(fail)))
  122. def addError(self, test, error):
  123. """
  124. Report an error that occurred while running the given test.
  125. @type test: L{pyunit.TestCase}
  126. @type error: L{Failure} or L{tuple}
  127. """
  128. self.errors.append((test, self._getFailure(error)))
  129. def addSkip(self, test, reason):
  130. """
  131. Report that the given test was skipped.
  132. In Trial, tests can be 'skipped'. Tests are skipped mostly because
  133. there is some platform or configuration issue that prevents them from
  134. being run correctly.
  135. @type test: L{pyunit.TestCase}
  136. @type reason: L{str}
  137. """
  138. self.skips.append((test, reason))
  139. def addUnexpectedSuccess(self, test, todo=None):
  140. """
  141. Report that the given test succeeded against expectations.
  142. In Trial, tests can be marked 'todo'. That is, they are expected to
  143. fail. When a test that is expected to fail instead succeeds, it should
  144. call this method to report the unexpected success.
  145. @type test: L{pyunit.TestCase}
  146. @type todo: L{unittest.Todo}, or L{None}, in which case a default todo
  147. message is provided.
  148. """
  149. if todo is None:
  150. todo = _makeTodo(self._DEFAULT_TODO)
  151. self.unexpectedSuccesses.append((test, todo))
  152. def addExpectedFailure(self, test, error, todo=None):
  153. """
  154. Report that the given test failed, and was expected to do so.
  155. In Trial, tests can be marked 'todo'. That is, they are expected to
  156. fail.
  157. @type test: L{pyunit.TestCase}
  158. @type error: L{Failure}
  159. @type todo: L{unittest.Todo}, or L{None}, in which case a default todo
  160. message is provided.
  161. """
  162. if todo is None:
  163. todo = _makeTodo(self._DEFAULT_TODO)
  164. self.expectedFailures.append((test, error, todo))
  165. def addSuccess(self, test):
  166. """
  167. Report that the given test succeeded.
  168. @type test: L{pyunit.TestCase}
  169. """
  170. self.successes += 1
  171. def wasSuccessful(self):
  172. """
  173. Report whether or not this test suite was successful or not.
  174. The behaviour of this method changed in L{pyunit} in Python 3.4 to
  175. fail if there are any errors, failures, or unexpected successes.
  176. Previous to 3.4, it was only if there were errors or failures. This
  177. method implements the old behaviour for backwards compatibility reasons,
  178. checking just for errors and failures.
  179. @rtype: L{bool}
  180. """
  181. return len(self.failures) == len(self.errors) == 0
  182. def done(self):
  183. """
  184. The test suite has finished running.
  185. """
  186. @implementer(itrial.IReporter)
  187. class TestResultDecorator(
  188. proxyForInterface(itrial.IReporter, "_originalReporter") # type: ignore[misc]
  189. ):
  190. """
  191. Base class for TestResult decorators.
  192. @ivar _originalReporter: The wrapped instance of reporter.
  193. @type _originalReporter: A provider of L{itrial.IReporter}
  194. """
  195. @implementer(itrial.IReporter)
  196. class UncleanWarningsReporterWrapper(TestResultDecorator):
  197. """
  198. A wrapper for a reporter that converts L{util.DirtyReactorAggregateError}s
  199. to warnings.
  200. """
  201. def addError(self, test, error):
  202. """
  203. If the error is a L{util.DirtyReactorAggregateError}, instead of
  204. reporting it as a normal error, throw a warning.
  205. """
  206. if isinstance(error, Failure) and error.check(util.DirtyReactorAggregateError):
  207. warnings.warn(error.getErrorMessage())
  208. else:
  209. self._originalReporter.addError(test, error)
  210. @implementer(itrial.IReporter)
  211. class _ExitWrapper(TestResultDecorator):
  212. """
  213. A wrapper for a reporter that causes the reporter to stop after
  214. unsuccessful tests.
  215. """
  216. def addError(self, *args, **kwargs):
  217. self.shouldStop = True
  218. return self._originalReporter.addError(*args, **kwargs)
  219. def addFailure(self, *args, **kwargs):
  220. self.shouldStop = True
  221. return self._originalReporter.addFailure(*args, **kwargs)
  222. class _AdaptedReporter(TestResultDecorator):
  223. """
  224. TestResult decorator that makes sure that addError only gets tests that
  225. have been adapted with a particular test adapter.
  226. """
  227. def __init__(self, original, testAdapter):
  228. """
  229. Construct an L{_AdaptedReporter}.
  230. @param original: An {itrial.IReporter}.
  231. @param testAdapter: A callable that returns an L{itrial.ITestCase}.
  232. """
  233. TestResultDecorator.__init__(self, original)
  234. self.testAdapter = testAdapter
  235. def addError(self, test, error):
  236. """
  237. See L{itrial.IReporter}.
  238. """
  239. test = self.testAdapter(test)
  240. return self._originalReporter.addError(test, error)
  241. def addExpectedFailure(self, test, failure, todo=None):
  242. """
  243. See L{itrial.IReporter}.
  244. @type test: A L{pyunit.TestCase}.
  245. @type failure: A L{failure.Failure} or L{AssertionError}
  246. @type todo: A L{unittest.Todo} or None
  247. When C{todo} is L{None} a generic C{unittest.Todo} is built.
  248. L{pyunit.TestCase}'s C{run()} calls this with 3 positional arguments
  249. (without C{todo}).
  250. """
  251. return self._originalReporter.addExpectedFailure(
  252. self.testAdapter(test), failure, todo
  253. )
  254. def addFailure(self, test, failure):
  255. """
  256. See L{itrial.IReporter}.
  257. """
  258. test = self.testAdapter(test)
  259. return self._originalReporter.addFailure(test, failure)
  260. def addSkip(self, test, skip):
  261. """
  262. See L{itrial.IReporter}.
  263. """
  264. test = self.testAdapter(test)
  265. return self._originalReporter.addSkip(test, skip)
  266. def addUnexpectedSuccess(self, test, todo=None):
  267. """
  268. See L{itrial.IReporter}.
  269. @type test: A L{pyunit.TestCase}.
  270. @type todo: A L{unittest.Todo} or None
  271. When C{todo} is L{None} a generic C{unittest.Todo} is built.
  272. L{pyunit.TestCase}'s C{run()} calls this with 2 positional arguments
  273. (without C{todo}).
  274. """
  275. test = self.testAdapter(test)
  276. return self._originalReporter.addUnexpectedSuccess(test, todo)
  277. def startTest(self, test):
  278. """
  279. See L{itrial.IReporter}.
  280. """
  281. return self._originalReporter.startTest(self.testAdapter(test))
  282. def stopTest(self, test):
  283. """
  284. See L{itrial.IReporter}.
  285. """
  286. return self._originalReporter.stopTest(self.testAdapter(test))
  287. @implementer(itrial.IReporter)
  288. class Reporter(TestResult):
  289. """
  290. A basic L{TestResult} with support for writing to a stream.
  291. @ivar _startTime: The time when the first test was started. It defaults to
  292. L{None}, which means that no test was actually launched.
  293. @type _startTime: C{float} or L{None}
  294. @ivar _warningCache: A C{set} of tuples of warning message (file, line,
  295. text, category) which have already been written to the output stream
  296. during the currently executing test. This is used to avoid writing
  297. duplicates of the same warning to the output stream.
  298. @type _warningCache: C{set}
  299. @ivar _publisher: The log publisher which will be observed for warning
  300. events.
  301. @type _publisher: L{twisted.python.log.LogPublisher}
  302. """
  303. _separator = "-" * 79
  304. _doubleSeparator = "=" * 79
  305. def __init__(
  306. self, stream=sys.stdout, tbformat="default", realtime=False, publisher=None
  307. ):
  308. super().__init__()
  309. self._stream = SafeStream(stream)
  310. self.tbformat = tbformat
  311. self.realtime = realtime
  312. self._startTime = None
  313. self._warningCache = set()
  314. # Start observing log events so as to be able to report warnings.
  315. self._publisher = publisher
  316. if publisher is not None:
  317. publisher.addObserver(self._observeWarnings)
  318. def _observeWarnings(self, event):
  319. """
  320. Observe warning events and write them to C{self._stream}.
  321. This method is a log observer which will be registered with
  322. C{self._publisher.addObserver}.
  323. @param event: A C{dict} from the logging system. If it has a
  324. C{'warning'} key, a logged warning will be extracted from it and
  325. possibly written to C{self.stream}.
  326. """
  327. if "warning" in event:
  328. key = (
  329. event["filename"],
  330. event["lineno"],
  331. event["category"].split(".")[-1],
  332. str(event["warning"]),
  333. )
  334. if key not in self._warningCache:
  335. self._warningCache.add(key)
  336. self._stream.write("%s:%s: %s: %s\n" % key)
  337. def startTest(self, test):
  338. """
  339. Called when a test begins to run. Records the time when it was first
  340. called and resets the warning cache.
  341. @param test: L{ITestCase}
  342. """
  343. super().startTest(test)
  344. if self._startTime is None:
  345. self._startTime = self._getTime()
  346. self._warningCache = set()
  347. def addFailure(self, test, fail):
  348. """
  349. Called when a test fails. If C{realtime} is set, then it prints the
  350. error to the stream.
  351. @param test: L{ITestCase} that failed.
  352. @param fail: L{failure.Failure} containing the error.
  353. """
  354. super().addFailure(test, fail)
  355. if self.realtime:
  356. fail = self.failures[-1][1] # guarantee it's a Failure
  357. self._write(self._formatFailureTraceback(fail))
  358. def addError(self, test, error):
  359. """
  360. Called when a test raises an error. If C{realtime} is set, then it
  361. prints the error to the stream.
  362. @param test: L{ITestCase} that raised the error.
  363. @param error: L{failure.Failure} containing the error.
  364. """
  365. error = self._getFailure(error)
  366. super().addError(test, error)
  367. if self.realtime:
  368. error = self.errors[-1][1] # guarantee it's a Failure
  369. self._write(self._formatFailureTraceback(error))
  370. def _write(self, format, *args):
  371. """
  372. Safely write to the reporter's stream.
  373. @param format: A format string to write.
  374. @param args: The arguments for the format string.
  375. """
  376. s = str(format)
  377. assert isinstance(s, str)
  378. if args:
  379. self._stream.write(s % args)
  380. else:
  381. self._stream.write(s)
  382. untilConcludes(self._stream.flush)
  383. def _writeln(self, format, *args):
  384. """
  385. Safely write a line to the reporter's stream. Newline is appended to
  386. the format string.
  387. @param format: A format string to write.
  388. @param args: The arguments for the format string.
  389. """
  390. self._write(format, *args)
  391. self._write("\n")
  392. def upDownError(self, method, error, warn=True, printStatus=True):
  393. super().upDownError(method, error, warn, printStatus)
  394. if warn:
  395. tbStr = self._formatFailureTraceback(error)
  396. log.msg(tbStr)
  397. msg = "caught exception in {}, your TestCase is broken\n\n{}".format(
  398. method,
  399. tbStr,
  400. )
  401. warnings.warn(msg, BrokenTestCaseWarning, stacklevel=2)
  402. def cleanupErrors(self, errs):
  403. super().cleanupErrors(errs)
  404. warnings.warn(
  405. "%s\n%s"
  406. % (
  407. "REACTOR UNCLEAN! traceback(s) follow: ",
  408. self._formatFailureTraceback(errs),
  409. ),
  410. BrokenTestCaseWarning,
  411. )
  412. def _trimFrames(self, frames):
  413. """
  414. Trim frames to remove internal paths.
  415. When a C{SynchronousTestCase} method fails synchronously, the stack
  416. looks like this:
  417. - [0]: C{SynchronousTestCase._run}
  418. - [1]: C{util.runWithWarningsSuppressed}
  419. - [2:-2]: code in the test method which failed
  420. - [-1]: C{_synctest.fail}
  421. When a C{TestCase} method fails synchronously, the stack looks like
  422. this:
  423. - [0]: C{defer.maybeDeferred}
  424. - [1]: C{utils.runWithWarningsSuppressed}
  425. - [2]: C{utils.runWithWarningsSuppressed}
  426. - [3:-2]: code in the test method which failed
  427. - [-1]: C{_synctest.fail}
  428. When a method fails inside a C{Deferred} (i.e., when the test method
  429. returns a C{Deferred}, and that C{Deferred}'s errback fires), the stack
  430. captured inside the resulting C{Failure} looks like this:
  431. - [0]: C{defer.Deferred._runCallbacks}
  432. - [1:-2]: code in the testmethod which failed
  433. - [-1]: C{_synctest.fail}
  434. As a result, we want to trim either [maybeDeferred, runWWS, runWWS] or
  435. [Deferred._runCallbacks] or [SynchronousTestCase._run, runWWS] from the
  436. front, and trim the [unittest.fail] from the end.
  437. There is also another case, when the test method is badly defined and
  438. contains extra arguments.
  439. If it doesn't recognize one of these cases, it just returns the
  440. original frames.
  441. @param frames: The C{list} of frames from the test failure.
  442. @return: The C{list} of frames to display.
  443. """
  444. newFrames = list(frames)
  445. if len(frames) < 2:
  446. return newFrames
  447. firstMethod = newFrames[0][0]
  448. firstFile = os.path.splitext(os.path.basename(newFrames[0][1]))[0]
  449. secondMethod = newFrames[1][0]
  450. secondFile = os.path.splitext(os.path.basename(newFrames[1][1]))[0]
  451. syncCase = (("_run", "_synctest"), ("runWithWarningsSuppressed", "util"))
  452. asyncCase = (("maybeDeferred", "defer"), ("runWithWarningsSuppressed", "utils"))
  453. twoFrames = ((firstMethod, firstFile), (secondMethod, secondFile))
  454. # On PY3, we have an extra frame which is reraising the exception
  455. for frame in newFrames:
  456. frameFile = os.path.splitext(os.path.basename(frame[1]))[0]
  457. if frameFile == "compat" and frame[0] == "reraise":
  458. # If it's in the compat module and is reraise, BLAM IT
  459. newFrames.pop(newFrames.index(frame))
  460. if twoFrames == syncCase:
  461. newFrames = newFrames[2:]
  462. elif twoFrames == asyncCase:
  463. newFrames = newFrames[3:]
  464. elif (firstMethod, firstFile) == ("_runCallbacks", "defer"):
  465. newFrames = newFrames[1:]
  466. if not newFrames:
  467. # The method fails before getting called, probably an argument
  468. # problem
  469. return newFrames
  470. last = newFrames[-1]
  471. if (
  472. last[0].startswith("fail")
  473. and os.path.splitext(os.path.basename(last[1]))[0] == "_synctest"
  474. ):
  475. newFrames = newFrames[:-1]
  476. return newFrames
  477. def _formatFailureTraceback(self, fail):
  478. if isinstance(fail, str):
  479. return fail.rstrip() + "\n"
  480. fail.frames, frames = self._trimFrames(fail.frames), fail.frames
  481. result = fail.getTraceback(detail=self.tbformat, elideFrameworkCode=True)
  482. fail.frames = frames
  483. return result
  484. def _groupResults(self, results, formatter):
  485. """
  486. Group tests together based on their results.
  487. @param results: An iterable of tuples of two or more elements. The
  488. first element of each tuple is a test case. The remaining
  489. elements describe the outcome of that test case.
  490. @param formatter: A callable which turns a test case result into a
  491. string. The elements after the first of the tuples in
  492. C{results} will be passed as positional arguments to
  493. C{formatter}.
  494. @return: A C{list} of two-tuples. The first element of each tuple
  495. is a unique string describing one result from at least one of
  496. the test cases in C{results}. The second element is a list of
  497. the test cases which had that result.
  498. """
  499. groups = OrderedDict()
  500. for content in results:
  501. case = content[0]
  502. outcome = content[1:]
  503. key = formatter(*outcome)
  504. groups.setdefault(key, []).append(case)
  505. return list(groups.items())
  506. def _printResults(self, flavor, errors, formatter):
  507. """
  508. Print a group of errors to the stream.
  509. @param flavor: A string indicating the kind of error (e.g. 'TODO').
  510. @param errors: A list of errors, often L{failure.Failure}s, but
  511. sometimes 'todo' errors.
  512. @param formatter: A callable that knows how to format the errors.
  513. """
  514. for reason, cases in self._groupResults(errors, formatter):
  515. self._writeln(self._doubleSeparator)
  516. self._writeln(flavor)
  517. self._write(reason)
  518. self._writeln("")
  519. for case in cases:
  520. self._writeln(case.id())
  521. def _printExpectedFailure(self, error, todo):
  522. return "Reason: {!r}\n{}".format(
  523. todo.reason, self._formatFailureTraceback(error)
  524. )
  525. def _printUnexpectedSuccess(self, todo):
  526. ret = f"Reason: {todo.reason!r}\n"
  527. if todo.errors:
  528. ret += "Expected errors: {}\n".format(", ".join(todo.errors))
  529. return ret
  530. def _printErrors(self):
  531. """
  532. Print all of the non-success results to the stream in full.
  533. """
  534. self._write("\n")
  535. self._printResults("[SKIPPED]", self.skips, lambda x: "%s\n" % x)
  536. self._printResults("[TODO]", self.expectedFailures, self._printExpectedFailure)
  537. self._printResults("[FAIL]", self.failures, self._formatFailureTraceback)
  538. self._printResults("[ERROR]", self.errors, self._formatFailureTraceback)
  539. self._printResults(
  540. "[SUCCESS!?!]", self.unexpectedSuccesses, self._printUnexpectedSuccess
  541. )
  542. def _getSummary(self):
  543. """
  544. Return a formatted count of tests status results.
  545. """
  546. summaries = []
  547. for stat in (
  548. "skips",
  549. "expectedFailures",
  550. "failures",
  551. "errors",
  552. "unexpectedSuccesses",
  553. ):
  554. num = len(getattr(self, stat))
  555. if num:
  556. summaries.append("%s=%d" % (stat, num))
  557. if self.successes:
  558. summaries.append("successes=%d" % (self.successes,))
  559. summary = (summaries and " (" + ", ".join(summaries) + ")") or ""
  560. return summary
  561. def _printSummary(self):
  562. """
  563. Print a line summarising the test results to the stream.
  564. """
  565. summary = self._getSummary()
  566. if self.wasSuccessful():
  567. status = "PASSED"
  568. else:
  569. status = "FAILED"
  570. self._write("%s%s\n", status, summary)
  571. def done(self):
  572. """
  573. Summarize the result of the test run.
  574. The summary includes a report of all of the errors, todos, skips and
  575. so forth that occurred during the run. It also includes the number of
  576. tests that were run and how long it took to run them (not including
  577. load time).
  578. Expects that C{_printErrors}, C{_writeln}, C{_write}, C{_printSummary}
  579. and C{_separator} are all implemented.
  580. """
  581. if self._publisher is not None:
  582. self._publisher.removeObserver(self._observeWarnings)
  583. self._printErrors()
  584. self._writeln(self._separator)
  585. if self._startTime is not None:
  586. self._writeln(
  587. "Ran %d tests in %.3fs", self.testsRun, time.time() - self._startTime
  588. )
  589. self._write("\n")
  590. self._printSummary()
  591. class MinimalReporter(Reporter):
  592. """
  593. A minimalist reporter that prints only a summary of the test result, in
  594. the form of (timeTaken, #tests, #tests, #errors, #failures, #skips).
  595. """
  596. def _printErrors(self):
  597. """
  598. Don't print a detailed summary of errors. We only care about the
  599. counts.
  600. """
  601. def _printSummary(self):
  602. """
  603. Print out a one-line summary of the form:
  604. '%(runtime) %(number_of_tests) %(number_of_tests) %(num_errors)
  605. %(num_failures) %(num_skips)'
  606. """
  607. numTests = self.testsRun
  608. if self._startTime is not None:
  609. timing = self._getTime() - self._startTime
  610. else:
  611. timing = 0
  612. t = (
  613. timing,
  614. numTests,
  615. numTests,
  616. len(self.errors),
  617. len(self.failures),
  618. len(self.skips),
  619. )
  620. self._writeln(" ".join(map(str, t)))
  621. class TextReporter(Reporter):
  622. """
  623. Simple reporter that prints a single character for each test as it runs,
  624. along with the standard Trial summary text.
  625. """
  626. def addSuccess(self, test):
  627. super().addSuccess(test)
  628. self._write(".")
  629. def addError(self, *args):
  630. super().addError(*args)
  631. self._write("E")
  632. def addFailure(self, *args):
  633. super().addFailure(*args)
  634. self._write("F")
  635. def addSkip(self, *args):
  636. super().addSkip(*args)
  637. self._write("S")
  638. def addExpectedFailure(self, *args):
  639. super().addExpectedFailure(*args)
  640. self._write("T")
  641. def addUnexpectedSuccess(self, *args):
  642. super().addUnexpectedSuccess(*args)
  643. self._write("!")
  644. class VerboseTextReporter(Reporter):
  645. """
  646. A verbose reporter that prints the name of each test as it is running.
  647. Each line is printed with the name of the test, followed by the result of
  648. that test.
  649. """
  650. # This is actually the bwverbose option
  651. def startTest(self, tm):
  652. self._write("%s ... ", tm.id())
  653. super().startTest(tm)
  654. def addSuccess(self, test):
  655. super().addSuccess(test)
  656. self._write("[OK]")
  657. def addError(self, *args):
  658. super().addError(*args)
  659. self._write("[ERROR]")
  660. def addFailure(self, *args):
  661. super().addFailure(*args)
  662. self._write("[FAILURE]")
  663. def addSkip(self, *args):
  664. super().addSkip(*args)
  665. self._write("[SKIPPED]")
  666. def addExpectedFailure(self, *args):
  667. super().addExpectedFailure(*args)
  668. self._write("[TODO]")
  669. def addUnexpectedSuccess(self, *args):
  670. super().addUnexpectedSuccess(*args)
  671. self._write("[SUCCESS!?!]")
  672. def stopTest(self, test):
  673. super().stopTest(test)
  674. self._write("\n")
  675. class TimingTextReporter(VerboseTextReporter):
  676. """
  677. Prints out each test as it is running, followed by the time taken for each
  678. test to run.
  679. """
  680. def stopTest(self, method):
  681. """
  682. Mark the test as stopped, and write the time it took to run the test
  683. to the stream.
  684. """
  685. super().stopTest(method)
  686. self._write("(%.03f secs)\n" % self._lastTime)
  687. class _AnsiColorizer:
  688. """
  689. A colorizer is an object that loosely wraps around a stream, allowing
  690. callers to write text to the stream in a particular color.
  691. Colorizer classes must implement C{supported()} and C{write(text, color)}.
  692. """
  693. _colors = dict(
  694. black=30, red=31, green=32, yellow=33, blue=34, magenta=35, cyan=36, white=37
  695. )
  696. def __init__(self, stream):
  697. self.stream = stream
  698. @classmethod
  699. def supported(cls, stream=sys.stdout):
  700. """
  701. A class method that returns True if the current platform supports
  702. coloring terminal output using this method. Returns False otherwise.
  703. """
  704. if not stream.isatty():
  705. return False # auto color only on TTYs
  706. try:
  707. import curses
  708. except ImportError:
  709. return False
  710. else:
  711. try:
  712. try:
  713. return curses.tigetnum("colors") > 2
  714. except curses.error:
  715. curses.setupterm()
  716. return curses.tigetnum("colors") > 2
  717. except BaseException:
  718. # guess false in case of error
  719. return False
  720. def write(self, text, color):
  721. """
  722. Write the given text to the stream in the given color.
  723. @param text: Text to be written to the stream.
  724. @param color: A string label for a color. e.g. 'red', 'white'.
  725. """
  726. color = self._colors[color]
  727. self.stream.write(f"\x1b[{color};1m{text}\x1b[0m")
  728. class _Win32Colorizer:
  729. """
  730. See _AnsiColorizer docstring.
  731. """
  732. def __init__(self, stream):
  733. from win32console import ( # type: ignore[import]
  734. FOREGROUND_BLUE,
  735. FOREGROUND_GREEN,
  736. FOREGROUND_INTENSITY,
  737. FOREGROUND_RED,
  738. STD_OUTPUT_HANDLE,
  739. GetStdHandle,
  740. )
  741. red, green, blue, bold = (
  742. FOREGROUND_RED,
  743. FOREGROUND_GREEN,
  744. FOREGROUND_BLUE,
  745. FOREGROUND_INTENSITY,
  746. )
  747. self.stream = stream
  748. self.screenBuffer = GetStdHandle(STD_OUTPUT_HANDLE)
  749. self._colors = {
  750. "normal": red | green | blue,
  751. "red": red | bold,
  752. "green": green | bold,
  753. "blue": blue | bold,
  754. "yellow": red | green | bold,
  755. "magenta": red | blue | bold,
  756. "cyan": green | blue | bold,
  757. "white": red | green | blue | bold,
  758. }
  759. @classmethod
  760. def supported(cls, stream=sys.stdout):
  761. try:
  762. import win32console
  763. screenBuffer = win32console.GetStdHandle(win32console.STD_OUTPUT_HANDLE)
  764. except ImportError:
  765. return False
  766. import pywintypes # type: ignore[import]
  767. try:
  768. screenBuffer.SetConsoleTextAttribute(
  769. win32console.FOREGROUND_RED
  770. | win32console.FOREGROUND_GREEN
  771. | win32console.FOREGROUND_BLUE
  772. )
  773. except pywintypes.error:
  774. return False
  775. else:
  776. return True
  777. def write(self, text, color):
  778. color = self._colors[color]
  779. self.screenBuffer.SetConsoleTextAttribute(color)
  780. self.stream.write(text)
  781. self.screenBuffer.SetConsoleTextAttribute(self._colors["normal"])
  782. class _NullColorizer:
  783. """
  784. See _AnsiColorizer docstring.
  785. """
  786. def __init__(self, stream):
  787. self.stream = stream
  788. @classmethod
  789. def supported(cls, stream=sys.stdout):
  790. return True
  791. def write(self, text, color):
  792. self.stream.write(text)
  793. @implementer(itrial.IReporter)
  794. class SubunitReporter:
  795. """
  796. Reports test output via Subunit.
  797. @ivar _subunit: The subunit protocol client that we are wrapping.
  798. @ivar _successful: An internal variable, used to track whether we have
  799. received only successful results.
  800. @since: 10.0
  801. """
  802. testsRun = None
  803. def __init__(
  804. self, stream=sys.stdout, tbformat="default", realtime=False, publisher=None
  805. ):
  806. """
  807. Construct a L{SubunitReporter}.
  808. @param stream: A file-like object representing the stream to print
  809. output to. Defaults to stdout.
  810. @param tbformat: The format for tracebacks. Ignored, since subunit
  811. always uses Python's standard format.
  812. @param realtime: Whether or not to print exceptions in the middle
  813. of the test results. Ignored, since subunit always does this.
  814. @param publisher: The log publisher which will be preserved for
  815. reporting events. Ignored, as it's not relevant to subunit.
  816. """
  817. if TestProtocolClient is None:
  818. raise Exception("Subunit not available")
  819. self._subunit = TestProtocolClient(stream)
  820. self._successful = True
  821. def done(self):
  822. """
  823. Record that the entire test suite run is finished.
  824. We do nothing, since a summary clause is irrelevant to the subunit
  825. protocol.
  826. """
  827. pass
  828. @property
  829. def shouldStop(self):
  830. """
  831. Whether or not the test runner should stop running tests.
  832. """
  833. return self._subunit.shouldStop
  834. def stop(self):
  835. """
  836. Signal that the test runner should stop running tests.
  837. """
  838. return self._subunit.stop()
  839. def wasSuccessful(self):
  840. """
  841. Has the test run been successful so far?
  842. @return: C{True} if we have received no reports of errors or failures,
  843. C{False} otherwise.
  844. """
  845. # Subunit has a bug in its implementation of wasSuccessful, see
  846. # https://bugs.edge.launchpad.net/subunit/+bug/491090, so we can't
  847. # simply forward it on.
  848. return self._successful
  849. def startTest(self, test):
  850. """
  851. Record that C{test} has started.
  852. """
  853. return self._subunit.startTest(test)
  854. def stopTest(self, test):
  855. """
  856. Record that C{test} has completed.
  857. """
  858. return self._subunit.stopTest(test)
  859. def addSuccess(self, test):
  860. """
  861. Record that C{test} was successful.
  862. """
  863. return self._subunit.addSuccess(test)
  864. def addSkip(self, test, reason):
  865. """
  866. Record that C{test} was skipped for C{reason}.
  867. Some versions of subunit don't have support for addSkip. In those
  868. cases, the skip is reported as a success.
  869. @param test: A unittest-compatible C{TestCase}.
  870. @param reason: The reason for it being skipped. The C{str()} of this
  871. object will be included in the subunit output stream.
  872. """
  873. addSkip = getattr(self._subunit, "addSkip", None)
  874. if addSkip is None:
  875. self.addSuccess(test)
  876. else:
  877. self._subunit.addSkip(test, reason)
  878. def addError(self, test, err):
  879. """
  880. Record that C{test} failed with an unexpected error C{err}.
  881. Also marks the run as being unsuccessful, causing
  882. L{SubunitReporter.wasSuccessful} to return C{False}.
  883. """
  884. self._successful = False
  885. return self._subunit.addError(test, util.excInfoOrFailureToExcInfo(err))
  886. def addFailure(self, test, err):
  887. """
  888. Record that C{test} failed an assertion with the error C{err}.
  889. Also marks the run as being unsuccessful, causing
  890. L{SubunitReporter.wasSuccessful} to return C{False}.
  891. """
  892. self._successful = False
  893. return self._subunit.addFailure(test, util.excInfoOrFailureToExcInfo(err))
  894. def addExpectedFailure(self, test, failure, todo=None):
  895. """
  896. Record an expected failure from a test.
  897. Some versions of subunit do not implement this. For those versions, we
  898. record a success.
  899. """
  900. failure = util.excInfoOrFailureToExcInfo(failure)
  901. addExpectedFailure = getattr(self._subunit, "addExpectedFailure", None)
  902. if addExpectedFailure is None:
  903. self.addSuccess(test)
  904. else:
  905. addExpectedFailure(test, failure)
  906. def addUnexpectedSuccess(self, test, todo=None):
  907. """
  908. Record an unexpected success.
  909. Since subunit has no way of expressing this concept, we record a
  910. success on the subunit stream.
  911. """
  912. # Not represented in pyunit/subunit.
  913. self.addSuccess(test)
  914. class TreeReporter(Reporter):
  915. """
  916. Print out the tests in the form a tree.
  917. Tests are indented according to which class and module they belong.
  918. Results are printed in ANSI color.
  919. """
  920. currentLine = ""
  921. indent = " "
  922. columns = 79
  923. FAILURE = "red"
  924. ERROR = "red"
  925. TODO = "blue"
  926. SKIP = "blue"
  927. TODONE = "red"
  928. SUCCESS = "green"
  929. def __init__(self, stream=sys.stdout, *args, **kwargs):
  930. super().__init__(stream, *args, **kwargs)
  931. self._lastTest = []
  932. for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
  933. if colorizer.supported(stream):
  934. self._colorizer = colorizer(stream)
  935. break
  936. def getDescription(self, test):
  937. """
  938. Return the name of the method which 'test' represents. This is
  939. what gets displayed in the leaves of the tree.
  940. e.g. getDescription(TestCase('test_foo')) ==> test_foo
  941. """
  942. return test.id().split(".")[-1]
  943. def addSuccess(self, test):
  944. super().addSuccess(test)
  945. self.endLine("[OK]", self.SUCCESS)
  946. def addError(self, *args):
  947. super().addError(*args)
  948. self.endLine("[ERROR]", self.ERROR)
  949. def addFailure(self, *args):
  950. super().addFailure(*args)
  951. self.endLine("[FAIL]", self.FAILURE)
  952. def addSkip(self, *args):
  953. super().addSkip(*args)
  954. self.endLine("[SKIPPED]", self.SKIP)
  955. def addExpectedFailure(self, *args):
  956. super().addExpectedFailure(*args)
  957. self.endLine("[TODO]", self.TODO)
  958. def addUnexpectedSuccess(self, *args):
  959. super().addUnexpectedSuccess(*args)
  960. self.endLine("[SUCCESS!?!]", self.TODONE)
  961. def _write(self, format, *args):
  962. if args:
  963. format = format % args
  964. self.currentLine = format
  965. super()._write(self.currentLine)
  966. def _getPreludeSegments(self, testID):
  967. """
  968. Return a list of all non-leaf segments to display in the tree.
  969. Normally this is the module and class name.
  970. """
  971. segments = testID.split(".")[:-1]
  972. if len(segments) == 0:
  973. return segments
  974. segments = [
  975. seg for seg in (".".join(segments[:-1]), segments[-1]) if len(seg) > 0
  976. ]
  977. return segments
  978. def _testPrelude(self, testID):
  979. """
  980. Write the name of the test to the stream, indenting it appropriately.
  981. If the test is the first test in a new 'branch' of the tree, also
  982. write all of the parents in that branch.
  983. """
  984. segments = self._getPreludeSegments(testID)
  985. indentLevel = 0
  986. for seg in segments:
  987. if indentLevel < len(self._lastTest):
  988. if seg != self._lastTest[indentLevel]:
  989. self._write(f"{self.indent * indentLevel}{seg}\n")
  990. else:
  991. self._write(f"{self.indent * indentLevel}{seg}\n")
  992. indentLevel += 1
  993. self._lastTest = segments
  994. def cleanupErrors(self, errs):
  995. self._colorizer.write(" cleanup errors", self.ERROR)
  996. self.endLine("[ERROR]", self.ERROR)
  997. super().cleanupErrors(errs)
  998. def upDownError(self, method, error, warn, printStatus):
  999. self._colorizer.write(" %s" % method, self.ERROR)
  1000. if printStatus:
  1001. self.endLine("[ERROR]", self.ERROR)
  1002. super().upDownError(method, error, warn, printStatus)
  1003. def startTest(self, test):
  1004. """
  1005. Called when C{test} starts. Writes the tests name to the stream using
  1006. a tree format.
  1007. """
  1008. self._testPrelude(test.id())
  1009. self._write(
  1010. "%s%s ... "
  1011. % (self.indent * (len(self._lastTest)), self.getDescription(test))
  1012. )
  1013. super().startTest(test)
  1014. def endLine(self, message, color):
  1015. """
  1016. Print 'message' in the given color.
  1017. @param message: A string message, usually '[OK]' or something similar.
  1018. @param color: A string color, 'red', 'green' and so forth.
  1019. """
  1020. spaces = " " * (self.columns - len(self.currentLine) - len(message))
  1021. super()._write(spaces)
  1022. self._colorizer.write(message, color)
  1023. super()._write("\n")
  1024. def _printSummary(self):
  1025. """
  1026. Print a line summarising the test results to the stream, and color the
  1027. status result.
  1028. """
  1029. summary = self._getSummary()
  1030. if self.wasSuccessful():
  1031. status = "PASSED"
  1032. color = self.SUCCESS
  1033. else:
  1034. status = "FAILED"
  1035. color = self.FAILURE
  1036. self._colorizer.write(status, color)
  1037. self._write("%s\n", summary)