testresult.py 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204
  1. '''Test runner and result class for the regression test suite.
  2. '''
  3. import functools
  4. import io
  5. import sys
  6. import time
  7. import traceback
  8. import unittest
  9. import xml.etree.ElementTree as ET
  10. from datetime import datetime
  11. class RegressionTestResult(unittest.TextTestResult):
  12. separator1 = '=' * 70 + '\n'
  13. separator2 = '-' * 70 + '\n'
  14. def __init__(self, stream, descriptions, verbosity):
  15. super().__init__(stream=stream, descriptions=descriptions, verbosity=0)
  16. self.buffer = True
  17. self.__suite = ET.Element('testsuite')
  18. self.__suite.set('start', datetime.utcnow().isoformat(' '))
  19. self.__e = None
  20. self.__start_time = None
  21. self.__results = []
  22. self.__verbose = bool(verbosity)
  23. @classmethod
  24. def __getId(cls, test):
  25. try:
  26. test_id = test.id
  27. except AttributeError:
  28. return str(test)
  29. try:
  30. return test_id()
  31. except TypeError:
  32. return str(test_id)
  33. return repr(test)
  34. def startTest(self, test):
  35. super().startTest(test)
  36. self.__e = e = ET.SubElement(self.__suite, 'testcase')
  37. self.__start_time = time.perf_counter()
  38. if self.__verbose:
  39. self.stream.write(f'{self.getDescription(test)} ... ')
  40. self.stream.flush()
  41. def _add_result(self, test, capture=False, **args):
  42. e = self.__e
  43. self.__e = None
  44. if e is None:
  45. return
  46. e.set('name', args.pop('name', self.__getId(test)))
  47. e.set('status', args.pop('status', 'run'))
  48. e.set('result', args.pop('result', 'completed'))
  49. if self.__start_time:
  50. e.set('time', f'{time.perf_counter() - self.__start_time:0.6f}')
  51. if capture:
  52. if self._stdout_buffer is not None:
  53. stdout = self._stdout_buffer.getvalue().rstrip()
  54. ET.SubElement(e, 'system-out').text = stdout
  55. if self._stderr_buffer is not None:
  56. stderr = self._stderr_buffer.getvalue().rstrip()
  57. ET.SubElement(e, 'system-err').text = stderr
  58. for k, v in args.items():
  59. if not k or not v:
  60. continue
  61. e2 = ET.SubElement(e, k)
  62. if hasattr(v, 'items'):
  63. for k2, v2 in v.items():
  64. if k2:
  65. e2.set(k2, str(v2))
  66. else:
  67. e2.text = str(v2)
  68. else:
  69. e2.text = str(v)
  70. def __write(self, c, word):
  71. if self.__verbose:
  72. self.stream.write(f'{word}\n')
  73. @classmethod
  74. def __makeErrorDict(cls, err_type, err_value, err_tb):
  75. if isinstance(err_type, type):
  76. if err_type.__module__ == 'builtins':
  77. typename = err_type.__name__
  78. else:
  79. typename = f'{err_type.__module__}.{err_type.__name__}'
  80. else:
  81. typename = repr(err_type)
  82. msg = traceback.format_exception(err_type, err_value, None)
  83. tb = traceback.format_exception(err_type, err_value, err_tb)
  84. return {
  85. 'type': typename,
  86. 'message': ''.join(msg),
  87. '': ''.join(tb),
  88. }
  89. def addError(self, test, err):
  90. self._add_result(test, True, error=self.__makeErrorDict(*err))
  91. super().addError(test, err)
  92. self.__write('E', 'ERROR')
  93. def addExpectedFailure(self, test, err):
  94. self._add_result(test, True, output=self.__makeErrorDict(*err))
  95. super().addExpectedFailure(test, err)
  96. self.__write('x', 'expected failure')
  97. def addFailure(self, test, err):
  98. self._add_result(test, True, failure=self.__makeErrorDict(*err))
  99. super().addFailure(test, err)
  100. self.__write('F', 'FAIL')
  101. def addSkip(self, test, reason):
  102. self._add_result(test, skipped=reason)
  103. super().addSkip(test, reason)
  104. self.__write('S', f'skipped {reason!r}')
  105. def addSuccess(self, test):
  106. self._add_result(test)
  107. super().addSuccess(test)
  108. self.__write('.', 'ok')
  109. def addUnexpectedSuccess(self, test):
  110. self._add_result(test, outcome='UNEXPECTED_SUCCESS')
  111. super().addUnexpectedSuccess(test)
  112. self.__write('u', 'unexpected success')
  113. def printErrors(self):
  114. if self.__verbose:
  115. self.stream.write('\n')
  116. self.printErrorList('ERROR', self.errors)
  117. self.printErrorList('FAIL', self.failures)
  118. def printErrorList(self, flavor, errors):
  119. for test, err in errors:
  120. self.stream.write(self.separator1)
  121. self.stream.write(f'{flavor}: {self.getDescription(test)}\n')
  122. self.stream.write(self.separator2)
  123. self.stream.write('%s\n' % err)
  124. def get_xml_element(self):
  125. e = self.__suite
  126. e.set('tests', str(self.testsRun))
  127. e.set('errors', str(len(self.errors)))
  128. e.set('failures', str(len(self.failures)))
  129. return e
  130. class QuietRegressionTestRunner:
  131. def __init__(self, stream, buffer=False):
  132. self.result = RegressionTestResult(stream, None, 0)
  133. self.result.buffer = buffer
  134. def run(self, test):
  135. test(self.result)
  136. return self.result
  137. def get_test_runner_class(verbosity, buffer=False):
  138. if verbosity:
  139. return functools.partial(unittest.TextTestRunner,
  140. resultclass=RegressionTestResult,
  141. buffer=buffer,
  142. verbosity=verbosity)
  143. return functools.partial(QuietRegressionTestRunner, buffer=buffer)
  144. def get_test_runner(stream, verbosity, capture_output=False):
  145. return get_test_runner_class(verbosity, capture_output)(stream)
  146. if __name__ == '__main__':
  147. class TestTests(unittest.TestCase):
  148. def test_pass(self):
  149. pass
  150. def test_pass_slow(self):
  151. time.sleep(1.0)
  152. def test_fail(self):
  153. print('stdout', file=sys.stdout)
  154. print('stderr', file=sys.stderr)
  155. self.fail('failure message')
  156. def test_error(self):
  157. print('stdout', file=sys.stdout)
  158. print('stderr', file=sys.stderr)
  159. raise RuntimeError('error message')
  160. suite = unittest.TestSuite()
  161. suite.addTest(unittest.makeSuite(TestTests))
  162. stream = io.StringIO()
  163. runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv))
  164. runner = runner_cls(sys.stdout)
  165. result = runner.run(suite)
  166. print('Output:', stream.getvalue())
  167. print('XML: ', end='')
  168. for s in ET.tostringlist(result.get_xml_element()):
  169. print(s.decode(), end='')
  170. print()