docs/notes/python-tests.rst
====
Test
====
.. contents:: Table of Contents
:backlinks: none
A simple Python unittest
------------------------
.. code-block:: python
# python unittests only run the function with prefix "test"
>>> from __future__ import print_function
>>> import unittest
>>> class TestFoo(unittest.TestCase):
... def test_foo(self):
... self.assertTrue(True)
... def fun_not_run(self):
... print("no run")
...
>>> unittest.main()
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
>>> import unittest
>>> class TestFail(unittest.TestCase):
... def test_false(self):
... self.assertTrue(False)
...
>>> unittest.main()
F
======================================================================
FAIL: test_false (__main__.TestFail)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<stdin>", line 3, in test_false
AssertionError: False is not true
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (failures=1)
Python unittest setup & teardown hierarchy
------------------------------------------
.. code-block:: python
from __future__ import print_function
import unittest
def fib(n):
return 1 if n<=2 else fib(n-1)+fib(n-2)
def setUpModule():
print("setup module")
def tearDownModule():
print("teardown module")
class TestFib(unittest.TestCase):
def setUp(self):
print("setUp")
self.n = 10
def tearDown(self):
print("tearDown")
del self.n
@classmethod
def setUpClass(cls):
print("setUpClass")
@classmethod
def tearDownClass(cls):
print("tearDownClass")
def test_fib_assert_equal(self):
self.assertEqual(fib(self.n), 55)
def test_fib_assert_true(self):
self.assertTrue(fib(self.n) == 55)
if __name__ == "__main__":
unittest.main()
output:
.. code-block:: console
$ python test.py
setup module
setUpClass
setUp
tearDown
.setUp
tearDown
.tearDownClass
teardown module
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
Different module of setUp & tearDown hierarchy
----------------------------------------------
.. code-block:: python
# test_module.py
from __future__ import print_function
import unittest
class TestFoo(unittest.TestCase):
@classmethod
def setUpClass(self):
print("foo setUpClass")
@classmethod
def tearDownClass(self):
print("foo tearDownClass")
def setUp(self):
print("foo setUp")
def tearDown(self):
print("foo tearDown")
def test_foo(self):
self.assertTrue(True)
class TestBar(unittest.TestCase):
def setUp(self):
print("bar setUp")
def tearDown(self):
print("bar tearDown")
def test_bar(self):
self.assertTrue(True)
# test.py
from __future__ import print_function
from test_module import TestFoo
from test_module import TestBar
import test_module
import unittest
def setUpModule():
print("setUpModule")
def tearDownModule():
print("tearDownModule")
if __name__ == "__main__":
test_module.setUpModule = setUpModule
test_module.tearDownModule = tearDownModule
suite1 = unittest.TestLoader().loadTestsFromTestCase(TestFoo)
suite2 = unittest.TestLoader().loadTestsFromTestCase(TestBar)
suite = unittest.TestSuite([suite1,suite2])
unittest.TextTestRunner().run(suite)
output:
.. code-block:: console
$ python test.py
setUpModule
foo setUpClass
foo setUp
foo tearDown
.foo tearDownClass
bar setUp
bar tearDown
.tearDownModule
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
Run tests via unittest.TextTestRunner
-------------------------------------
.. code-block:: python
>>> import unittest
>>> class TestFoo(unittest.TestCase):
... def test_foo(self):
... self.assertTrue(True)
... def test_bar(self):
... self.assertFalse(False)
>>> suite = unittest.TestLoader().loadTestsFromTestCase(TestFoo)
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_bar (__main__.TestFoo) ... ok
test_foo (__main__.TestFoo) ... ok
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
Test raise exception
--------------------
.. code-block:: python
>>> import unittest
>>> class TestRaiseException(unittest.TestCase):
... def test_raise_except(self):
... with self.assertRaises(SystemError):
... raise SystemError
>>> suite_loader = unittest.TestLoader()
>>> suite = suite_loader.loadTestsFromTestCase(TestRaiseException)
>>> unittest.TextTestRunner().run(suite)
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
>>> class TestRaiseFail(unittest.TestCase):
... def test_raise_fail(self):
... with self.assertRaises(SystemError):
... pass
>>> suite = unittest.TestLoader().loadTestsFromTestCase(TestRaiseFail)
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_raise_fail (__main__.TestRaiseFail) ... FAIL
======================================================================
FAIL: test_raise_fail (__main__.TestRaiseFail)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<stdin>", line 4, in test_raise_fail
AssertionError: SystemError not raised
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (failures=1)
Pass arguments into a TestCase
------------------------------
.. code-block:: python
>>> from __future__ import print_function
>>> import unittest
>>> class TestArg(unittest.TestCase):
... def __init__(self, testname, arg):
... super(TestArg, self).__init__(testname)
... self._arg = arg
... def setUp(self):
... print("setUp:", self._arg)
... def test_arg(self):
... print("test_arg:", self._arg)
... self.assertTrue(True)
...
>>> suite = unittest.TestSuite()
>>> suite.addTest(TestArg('test_arg', 'foo'))
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_arg (__main__.TestArg) ... setUp: foo
test_arg: foo
ok
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
Group multiple testcases into a suite
-------------------------------------
.. code-block:: python
>>> import unittest
>>> class TestFooBar(unittest.TestCase):
... def test_foo(self):
... self.assertTrue(True)
... def test_bar(self):
... self.assertTrue(True)
...
>>> class TestHelloWorld(unittest.TestCase):
... def test_hello(self):
... self.assertEqual("Hello", "Hello")
... def test_world(self):
... self.assertEqual("World", "World")
...
>>> suite_loader = unittest.TestLoader()
>>> suite1 = suite_loader.loadTestsFromTestCase(TestFooBar)
>>> suite2 = suite_loader.loadTestsFromTestCase(TestHelloWorld)
>>> suite = unittest.TestSuite([suite1, suite2])
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_bar (__main__.TestFooBar) ... ok
test_foo (__main__.TestFooBar) ... ok
test_hello (__main__.TestHelloWorld) ... ok
test_world (__main__.TestHelloWorld) ... ok
----------------------------------------------------------------------
Ran 4 tests in 0.000s
OK
Group multiple tests from different TestCase
--------------------------------------------
.. code-block:: python
>>> import unittest
>>> class TestFoo(unittest.TestCase):
... def test_foo(self):
... assert "foo" == "foo"
...
>>> class TestBar(unittest.TestCase):
... def test_bar(self):
... assert "bar" == "bar"
...
>>> suite = unittest.TestSuite()
>>> suite.addTest(TestFoo('test_foo'))
>>> suite.addTest(TestBar('test_bar'))
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_foo (__main__.TestFoo) ... ok
test_bar (__main__.TestBar) ... ok
----------------------------------------------------------------------
Ran 2 tests in 0.001s
OK
Skip some tests in the TestCase
-------------------------------
.. code-block:: python
>>> import unittest
>>> RUN_FOO = False
>>> DONT_RUN_BAR = False
>>> class TestSkip(unittest.TestCase):
... def test_always_run(self):
... self.assertTrue(True)
... @unittest.skip("always skip this test")
... def test_always_skip(self):
... raise RuntimeError
... @unittest.skipIf(RUN_FOO == False, "demo skipIf")
... def test_skipif(self):
... raise RuntimeError
... @unittest.skipUnless(DONT_RUN_BAR == True, "demo skipUnless")
... def test_skipunless(self):
... raise RuntimeError
...
>>> suite = unittest.TestLoader().loadTestsFromTestCase(TestSkip)
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_always_run (__main__.TestSkip) ... ok
test_always_skip (__main__.TestSkip) ... skipped 'always skip this test'
test_skipif (__main__.TestSkip) ... skipped 'demo skipIf'
test_skipunless (__main__.TestSkip) ... skipped 'demo skipUnless'
----------------------------------------------------------------------
Ran 4 tests in 0.000s
OK (skipped=3)
Monolithic Test
----------------
.. code-block:: python
>>> from __future__ import print_function
>>> import unittest
>>> class Monolithic(unittest.TestCase):
... def step1(self):
... print('step1')
... def step2(self):
... print('step2')
... def step3(self):
... print('step3')
... def _steps(self):
... for attr in sorted(dir(self)):
... if not attr.startswith('step'):
... continue
... yield attr
... def test_foo(self):
... for _s in self._steps():
... try:
... getattr(self, _s)()
... except Exception as e:
... self.fail('{} failed({})'.format(attr, e))
...
>>> suite = unittest.TestLoader().loadTestsFromTestCase(Monolithic)
>>> unittest.TextTestRunner().run(suite)
step1
step2
step3
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
<unittest.runner.TextTestResult run=1 errors=0 failures=0>
Cross-module variables to Test files
------------------------------------
test_foo.py
.. code-block:: python
from __future__ import print_function
import unittest
print(conf)
class TestFoo(unittest.TestCase):
def test_foo(self):
print(conf)
@unittest.skipIf(conf.isskip==True, "skip test")
def test_skip(self):
raise RuntimeError
test_bar.py
.. code-block:: python
from __future__ import print_function
import unittest
import __builtin__
if __name__ == "__main__":
conf = type('TestConf', (object,), {})
conf.isskip = True
# make a cross-module variable
__builtin__.conf = conf
module = __import__('test_foo')
loader = unittest.TestLoader()
suite = loader.loadTestsFromTestCase(module.TestFoo)
unittest.TextTestRunner(verbosity=2).run(suite)
output:
.. code-block:: console
$ python test_bar.py
<class '__main__.TestConf'>
test_foo (test_foo.TestFoo) ... <class '__main__.TestConf'>
ok
test_skip (test_foo.TestFoo) ... skipped 'skip test'
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK (skipped=1)
skip setup & teardown when the test is skipped
-----------------------------------------------
.. code-block:: python
>>> from __future__ import print_function
>>> import unittest
>>> class TestSkip(unittest.TestCase):
... def setUp(self):
... print("setUp")
... def tearDown(self):
... print("tearDown")
... @unittest.skip("skip this test")
... def test_skip(self):
... raise RuntimeError
... def test_not_skip(self):
... self.assertTrue(True)
...
>>> suite = unittest.TestLoader().loadTestsFromTestCase(TestSkip)
>>> unittest.TextTestRunner(verbosity=2).run(suite)
test_not_skip (__main__.TestSkip) ... setUp
tearDown
ok
test_skip (__main__.TestSkip) ... skipped 'skip this test'
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK (skipped=1)
Re-using old test code
----------------------
.. code-block:: python
>>> from __future__ import print_function
>>> import unittest
>>> def old_func_test():
... assert "Hello" == "Hello"
...
>>> def old_func_setup():
... print("setup")
...
>>> def old_func_teardown():
... print("teardown")
...
>>> testcase = unittest.FunctionTestCase(old_func_test,
... setUp=old_func_setup,
... tearDown=old_func_teardown)
>>> suite = unittest.TestSuite([testcase])
>>> unittest.TextTestRunner().run(suite)
setup
teardown
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
<unittest.runner.TextTestResult run=1 errors=0 failures=0>
Testing your document is right
------------------------------
.. code-block:: python
"""
This is an example of doctest
>>> fib(10)
55
"""
def fib(n):
""" This function calculate fib number.
Example:
>>> fib(10)
55
>>> fib(-1)
Traceback (most recent call last):
...
ValueError
"""
if n < 0:
raise ValueError('')
return 1 if n<=2 else fib(n-1) + fib(n-2)
if __name__ == "__main__":
import doctest
doctest.testmod()
output:
.. code-block:: console
$ python demo_doctest.py -v
Trying:
fib(10)
Expecting:
55
ok
Trying:
fib(10)
Expecting:
55
ok
Trying:
fib(-1)
Expecting:
Traceback (most recent call last):
...
ValueError
ok
2 items passed all tests:
1 tests in __main__
2 tests in __main__.fib
3 tests in 2 items.
3 passed and 0 failed.
Test passed.
Re-using doctest to unittest
----------------------------
.. code-block:: python
import unittest
import doctest
"""
This is an example of doctest
>>> fib(10)
55
"""
def fib(n):
""" This function calculate fib number.
Example:
>>> fib(10)
55
>>> fib(-1)
Traceback (most recent call last):
...
ValueError
"""
if n < 0:
raise ValueError('')
return 1 if n<=2 else fib(n-1) + fib(n-2)
if __name__ == "__main__":
finder = doctest.DocTestFinder()
suite = doctest.DocTestSuite(test_finder=finder)
unittest.TextTestRunner(verbosity=2).run(suite)
output:
.. code-block:: console
fib (__main__)
Doctest: __main__.fib ... ok
----------------------------------------------------------------------
Ran 1 test in 0.023s
OK
Customize test report
----------------------
.. code-block:: python
from unittest import (
TestCase,
TestLoader,
TextTestResult,
TextTestRunner)
from pprint import pprint
import unittest
import os
OK = 'ok'
FAIL = 'fail'
ERROR = 'error'
SKIP = 'skip'
class JsonTestResult(TextTestResult):
def __init__(self, stream, descriptions, verbosity):
super_class = super(JsonTestResult, self)
super_class.__init__(stream, descriptions, verbosity)
# TextTestResult has no successes attr
self.successes = []
def addSuccess(self, test):
# addSuccess do nothing, so we need to overwrite it.
super(JsonTestResult, self).addSuccess(test)
self.successes.append(test)
def json_append(self, test, result, out):
suite = test.__class__.__name__
if suite not in out:
out[suite] = {OK: [], FAIL: [], ERROR:[], SKIP: []}
if result is OK:
out[suite][OK].append(test._testMethodName)
elif result is FAIL:
out[suite][FAIL].append(test._testMethodName)
elif result is ERROR:
out[suite][ERROR].append(test._testMethodName)
elif result is SKIP:
out[suite][SKIP].append(test._testMethodName)
else:
raise KeyError("No such result: {}".format(result))
return out
def jsonify(self):
json_out = dict()
for t in self.successes:
json_out = self.json_append(t, OK, json_out)
for t, _ in self.failures:
json_out = self.json_append(t, FAIL, json_out)
for t, _ in self.errors:
json_out = self.json_append(t, ERROR, json_out)
for t, _ in self.skipped:
json_out = self.json_append(t, SKIP, json_out)
return json_out
class TestSimple(TestCase):
def test_ok_1(self):
foo = True
self.assertTrue(foo)
def test_ok_2(self):
bar = True
self.assertTrue(bar)
def test_fail(self):
baz = False
self.assertTrue(baz)
def test_raise(self):
raise RuntimeError
@unittest.skip("Test skip")
def test_skip(self):
raise NotImplementedError
if __name__ == '__main__':
# redirector default output of unittest to /dev/null
with open(os.devnull, 'w') as null_stream:
# new a runner and overwrite resultclass of runner
runner = TextTestRunner(stream=null_stream)
runner.resultclass = JsonTestResult
# create a testsuite
suite = TestLoader().loadTestsFromTestCase(TestSimple)
# run the testsuite
result = runner.run(suite)
# print json output
pprint(result.jsonify())
output:
.. code-block:: bash
$ python test.py
{'TestSimple': {'error': ['test_raise'],
'fail': ['test_fail'],
'ok': ['test_ok_1', 'test_ok_2'],
'skip': ['test_skip']}}
Mock - using ``@patch`` substitute original method
----------------------------------------------------
.. code-block:: python
# python-3.3 or above
>>> from unittest.mock import patch
>>> import os
>>> def fake_remove(path, *a, **k):
... print("remove done")
...
>>> @patch('os.remove', fake_remove)
... def test():
... try:
... os.remove('%$!?&*') # fake os.remove
... except OSError as e:
... print(e)
... else:
... print('test success')
...
>>> test()
remove done
test success
.. note::
Without mock, above test will always fail.
.. code-block:: python
>>> import os
>>> def test():
... try:
... os.remove('%$!?&*')
... except OSError as e:
... print(e)
... else:
... print('test success')
...
>>> test()
[Errno 2] No such file or directory: '%$!?&*'
What ``with unittest.mock.patch`` do?
---------------------------------------
.. code-block:: python
from unittest.mock import patch
import os
PATH = '$@!%?&'
def fake_remove(path):
print("Fake remove")
class SimplePatch:
def __init__(self, target, new):
self._target = target
self._new = new
def get_target(self, target):
target, attr = target.rsplit('.', 1)
getter = __import__(target)
return getter, attr
def __enter__(self):
orig, attr = self.get_target(self._target)
self.orig, self.attr = orig, attr
self.orig_attr = getattr(orig, attr)
setattr(orig, attr, self._new)
return self._new
def __exit__(self, *exc_info):
setattr(self.orig, self.attr, self.orig_attr)
del self.orig_attr
print('---> inside unittest.mock.patch scope')
with patch('os.remove', fake_remove):
os.remove(PATH)
print('---> inside simple patch scope')
with SimplePatch('os.remove', fake_remove):
os.remove(PATH)
print('---> outside patch scope')
try:
os.remove(PATH)
except OSError as e:
print(e)
output:
.. code-block:: bash
$ python3 simple_patch.py
---> inside unittest.mock.patch scope
Fake remove
---> inside simple patch scope
Fake remove
---> outside patch scope
[Errno 2] No such file or directory: '$@!%?&'
Mock - substitute ``open``
---------------------------
.. code-block:: python
>>> import urllib
>>> from unittest.mock import patch, mock_open
>>> def send_req(url):
... with urllib.request.urlopen(url) as f:
... if f.status == 200:
... return f.read()
... raise urllib.error.URLError
...
>>> fake_html = b'<html><h1>Mock Content</h1></html>'
>>> mock_urlopen = mock_open(read_data=fake_html)
>>> ret = mock_urlopen.return_value
>>> ret.status = 200
>>> @patch('urllib.request.urlopen', mock_urlopen)
... def test_send_req_success():
... try:
... ret = send_req('http://www.mockurl.com')
... assert ret == fake_html
... except Exception as e:
... print(e)
... else:
... print('test send_req success')
...
>>> test_send_req_success()
test send_req success
>>> ret = mock_urlopen.return_value
>>> ret.status = 404
>>> @patch('urllib.request.urlopen', mock_urlopen)
... def test_send_req_fail():
... try:
... ret = send_req('http://www.mockurl.com')
... assert ret == fake_html
... except Exception as e:
... print('test fail success')
...
>>> test_send_req_fail()
test fail success