OpenStructure
xmlrunner.py
Go to the documentation of this file.
1 """
2 XML Test Runner for PyUnit
3 """
4 
5 # Written by Sebastian Rittau <srittau@jroger.in-berlin.de> and placed in
6 # the Public Domain. With contributions by Paolo Borelli and others.
7 
8 __version__ = "0.1"
9 
10 import os.path
11 import re
12 import sys
13 import time
14 import traceback
15 import unittest
16 from xml.sax.saxutils import escape
17 
18 try:
19  from io import StringIO
20 except ImportError:
21  from io import StringIO
22 
23 
24 class _TestInfo(object):
25 
26  """Information about a particular test.
27 
28  Used by _XMLTestResult.
29 
30  """
31 
32  def __init__(self, test, time):
33  (self._class, self._method) = test.id().rsplit(".", 1)
34  self._time_time = time
35  self._error_error = None
36  self._failure_failure = None
37 
38  @staticmethod
39  def create_success(test, time):
40  """Create a _TestInfo instance for a successful test."""
41  return _TestInfo(test, time)
42 
43  @staticmethod
44  def create_failure(test, time, failure):
45  """Create a _TestInfo instance for a failed test."""
46  info = _TestInfo(test, time)
47  info._failure = failure
48  return info
49 
50  @staticmethod
51  def create_error(test, time, error):
52  """Create a _TestInfo instance for an erroneous test."""
53  info = _TestInfo(test, time)
54  info._error = error
55  return info
56 
57  def print_report(self, stream):
58  """Print information about this test case in XML format to the
59  supplied stream.
60 
61  """
62  stream.write(' <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \
63  {
64  "class": self._class,
65  "method": self._method,
66  "time": self._time_time,
67  })
68  if self._failure_failure is not None:
69  self._print_error_print_error(stream, 'failure', self._failure_failure)
70  if self._error_error is not None:
71  self._print_error_print_error(stream, 'error', self._error_error)
72  stream.write('</testcase>\n')
73 
74  def _print_error(self, stream, tagname, error):
75  """Print information from a failure or error to the supplied stream."""
76  text = escape(str(error[1]))
77  stream.write('\n')
78  stream.write(' <%s type="%s">%s\n' \
79  % (tagname, _clsname(error[0]), text))
80  tb_stream = StringIO()
81  traceback.print_tb(error[2], None, tb_stream)
82  stream.write(escape(tb_stream.getvalue()))
83  stream.write(' </%s>\n' % tagname)
84  stream.write(' ')
85 
86 
87 def _clsname(cls):
88  return cls.__module__ + "." + cls.__name__
89 
90 
91 class _XMLTestResult(unittest.TestResult):
92 
93  """A test result class that stores result as XML.
94 
95  Used by XMLTestRunner.
96 
97  """
98 
99  def __init__(self, classname):
100  unittest.TestResult.__init__(self)
101  self._test_name_test_name = classname
102  self._start_time_start_time = None
103  self._tests_tests = []
104  self._error_error = None
105  self._failure_failure = None
106 
107  def startTest(self, test):
108  unittest.TestResult.startTest(self, test)
109  self._error_error = None
110  self._failure_failure = None
111  self._start_time_start_time = time.time()
112 
113  def stopTest(self, test):
114  time_taken = time.time() - self._start_time_start_time
115  unittest.TestResult.stopTest(self, test)
116  if self._error_error:
117  info = _TestInfo.create_error(test, time_taken, self._error_error)
118  elif self._failure_failure:
119  info = _TestInfo.create_failure(test, time_taken, self._failure_failure)
120  else:
121  info = _TestInfo.create_success(test, time_taken)
122  self._tests_tests.append(info)
123 
124  def addError(self, test, err):
125  unittest.TestResult.addError(self, test, err)
126  self._error_error = err
127 
128  def addFailure(self, test, err):
129  unittest.TestResult.addFailure(self, test, err)
130  self._failure_failure = err
131 
132  def print_report(self, stream, time_taken, out, err):
133  """Prints the XML report to the supplied stream.
134 
135  The time the tests took to perform as well as the captured standard
136  output and standard error streams must be passed in.a
137 
138  """
139  stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \
140  { "e": len(self.errors), "f": len(self.failures) })
141  stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \
142  {
143  "n": self._test_name_test_name,
144  "t": self.testsRun,
145  "time": time_taken,
146  })
147  for info in self._tests_tests:
148  info.print_report(stream)
149  stream.write(' <system-out><![CDATA[%s]]></system-out>\n' % out)
150  stream.write(' <system-err><![CDATA[%s]]></system-err>\n' % err)
151  stream.write('</testsuite>\n')
152 
153 
154 class XMLTestRunner(object):
155 
156  """A test runner that stores results in XML format compatible with JUnit.
157 
158  XMLTestRunner(stream=None) -> XML test runner
159 
160  The XML file is written to the supplied stream. If stream is None, the
161  results are stored in a file called TEST-<module>.<class>.xml in the
162  current working directory (if not overridden with the path property),
163  where <module> and <class> are the module and class name of the test class.
164 
165  """
166 
167  def __init__(self, stream=None):
168  self._stream_stream = stream
169  self._path_path = "."
170 
171  def run(self, test):
172  """Run the given test case or test suite."""
173  class_ = test.__class__
174  classname = class_.__module__ + "." + class_.__name__
175  if self._stream_stream == None:
176  filename = "TEST-%s.xml" % classname
177  stream = file(os.path.join(self._path_path, filename), "w")
178  stream.write('<?xml version="1.0" encoding="utf-8"?>\n')
179  else:
180  stream = self._stream_stream
181 
182  result = _XMLTestResult(classname)
183  start_time = time.time()
184 
185  try:
186  self._orig_stdout_orig_stdout = sys.stdout
187  self._orig_stderr_orig_stderr = sys.stderr
188  sys.stdout = StringIO()
189  sys.stderr = StringIO()
190  test(result)
191  try:
192  out_s = sys.stdout.getvalue()
193  except AttributeError:
194  out_s = ""
195  try:
196  err_s = sys.stderr.getvalue()
197  except AttributeError:
198  err_s = ""
199  finally:
200  sys.stdout = self._orig_stdout_orig_stdout
201  sys.stderr = self._orig_stderr_orig_stderr
202 
203 
204  time_taken = time.time() - start_time
205  result.print_report(stream, time_taken, out_s, err_s)
206  if self._stream_stream is None:
207  stream.close()
208 
209  return result
210 
211  def _set_path(self, path):
212  self._path_path = path
213 
214  path = property(lambda self: self._path_path, _set_path, None,
215  """The path where the XML files are stored.
216 
217  This property is ignored when the XML file is written to a file
218  stream.""")
219 
220 
221 class _fake_std_streams(object):
222 
223  def __enter__(self):
224  self._orig_stdout_orig_stdout = sys.stdout
225  self._orig_stderr_orig_stderr = sys.stderr
226  sys.stdout = StringIO()
227  sys.stderr = StringIO()
228 
229  def __exit__(self, exc_type, exc_val, exc_tb):
230  sys.stdout = self._orig_stdout_orig_stdout
231  sys.stderr = self._orig_stderr_orig_stderr
232 
233 
234 class XMLTestRunnerTest(unittest.TestCase):
235 
236  def setUp(self):
237  self._stream_stream = StringIO()
238 
239  def _try_test_run(self, test_class, expected):
240 
241  """Run the test suite against the supplied test class and compare the
242  XML result against the expected XML string. Fail if the expected
243  string doesn't match the actual string. All time attributes in the
244  expected string should have the value "0.000". All error and failure
245  messages are reduced to "Foobar".
246 
247  """
248 
249  runner = XMLTestRunner(self._stream_stream)
250  runner.run(unittest.makeSuite(test_class))
251 
252  got = self._stream_stream.getvalue()
253  # Replace all time="X.YYY" attributes by time="0.000" to enable a
254  # simple string comparison.
255  got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got)
256  # Likewise, replace all failure and error messages by a simple "Foobar"
257  # string.
258  got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got)
259  got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got)
260  # And finally Python 3 compatibility.
261  got = got.replace('type="builtins.', 'type="exceptions.')
262 
263  self.assertEqual(expected, got)
264 
265  def test_no_tests(self):
266  """Regression test: Check whether a test run without any tests
267  matches a previous run.
268 
269  """
270  class TestTest(unittest.TestCase):
271  pass
272  self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000">
273  <system-out><![CDATA[]]></system-out>
274  <system-err><![CDATA[]]></system-err>
275 </testsuite>
276 """)
277 
278  def test_success(self):
279  """Regression test: Check whether a test run with a successful test
280  matches a previous run.
281 
282  """
283  class TestTest(unittest.TestCase):
284  def test_foo(self):
285  pass
286  self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
287  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
288  <system-out><![CDATA[]]></system-out>
289  <system-err><![CDATA[]]></system-err>
290 </testsuite>
291 """)
292 
293  def test_failure(self):
294  """Regression test: Check whether a test run with a failing test
295  matches a previous run.
296 
297  """
298  class TestTest(unittest.TestCase):
299  def test_foo(self):
300  self.assertTrue(False)
301  self._try_test_run_try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000">
302  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
303  <failure type="exceptions.AssertionError">Foobar</failure>
304  </testcase>
305  <system-out><![CDATA[]]></system-out>
306  <system-err><![CDATA[]]></system-err>
307 </testsuite>
308 """)
309 
310  def test_error(self):
311  """Regression test: Check whether a test run with a erroneous test
312  matches a previous run.
313 
314  """
315  class TestTest(unittest.TestCase):
316  def test_foo(self):
317  raise IndexError()
318  self._try_test_run_try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
319  <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
320  <error type="exceptions.IndexError">Foobar</error>
321  </testcase>
322  <system-out><![CDATA[]]></system-out>
323  <system-err><![CDATA[]]></system-err>
324 </testsuite>
325 """)
326 
328  """Regression test: Check whether a test run with output to stdout
329  matches a previous run.
330 
331  """
332  class TestTest(unittest.TestCase):
333  def test_foo(self):
334  sys.stdout.write("Test\n")
335  self._try_test_run_try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
336  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
337  <system-out><![CDATA[Test
338 ]]></system-out>
339  <system-err><![CDATA[]]></system-err>
340 </testsuite>
341 """)
342 
344  """Regression test: Check whether a test run with output to stderr
345  matches a previous run.
346 
347  """
348  class TestTest(unittest.TestCase):
349  def test_foo(self):
350  sys.stderr.write("Test\n")
351  self._try_test_run_try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
352  <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
353  <system-out><![CDATA[]]></system-out>
354  <system-err><![CDATA[Test
355 ]]></system-err>
356 </testsuite>
357 """)
358 
359  class NullStream(object):
360  """A file-like object that discards everything written to it."""
361  def write(self, buffer):
362  pass
363 
365  """Check whether the XMLTestRunner recovers gracefully from unit tests
366  that change stdout, but don't change it back properly.
367 
368  """
369  class TestTest(unittest.TestCase):
370  def test_foo(self):
371  sys.stdout = XMLTestRunnerTest.NullStream()
372 
373  runner = XMLTestRunner(self._stream)
374  runner.run(unittest.makeSuite(TestTest))
375 
377  """Check whether the XMLTestRunner recovers gracefully from unit tests
378  that change stderr, but don't change it back properly.
379 
380  """
381  class TestTest(unittest.TestCase):
382  def test_foo(self):
383  sys.stderr = XMLTestRunnerTest.NullStream()
384 
385  runner = XMLTestRunner(self._stream_stream)
386  runner.run(unittest.makeSuite(TestTest))
387 
388 
389 if __name__ == "__main__":
390  unittest.main()
def create_success(test, time)
Definition: xmlrunner.py:39
def create_failure(test, time, failure)
Definition: xmlrunner.py:44
def create_error(test, time, error)
Definition: xmlrunner.py:51
def __init__(self, test, time)
Definition: xmlrunner.py:32
def _print_error(self, stream, tagname, error)
Definition: xmlrunner.py:74
def print_report(self, stream)
Definition: xmlrunner.py:57
def stopTest(self, test)
Definition: xmlrunner.py:113
def print_report(self, stream, time_taken, out, err)
Definition: xmlrunner.py:132
def addError(self, test, err)
Definition: xmlrunner.py:124
def __init__(self, classname)
Definition: xmlrunner.py:99
def startTest(self, test)
Definition: xmlrunner.py:107
def addFailure(self, test, err)
Definition: xmlrunner.py:128
def __exit__(self, exc_type, exc_val, exc_tb)
Definition: xmlrunner.py:229
def __init__(self, stream=None)
Definition: xmlrunner.py:167
def test_unittests_changing_stderr(self)
Definition: xmlrunner.py:376
def _try_test_run(self, test_class, expected)
Definition: xmlrunner.py:239
def test_unittests_changing_stdout(self)
Definition: xmlrunner.py:364