py.test allows to easily implement your own custom parametrization scheme for tests. Here we provide some examples for inspiration and re-use.
Let’s say we want to execute a test with different parameters and the parameter range shall be determined by a command line argument. Let’s first write a simple computation test:
# content of test_compute.py
def test_compute(param1):
assert param1 < 4
Now we add a test configuration like this:
# content of conftest.py
def pytest_addoption(parser):
parser.addoption("--all", action="store_true",
help="run all combinations")
def pytest_generate_tests(metafunc):
if 'param1' in metafunc.funcargnames:
if metafunc.config.option.all:
end = 5
else:
end = 2
for i in range(end):
metafunc.addcall(funcargs={'param1': i})
This means that we only run 2 tests if we do not pass --all:
$ py.test -q test_compute.py
collecting ... collected 2 items
..
2 passed in 0.01 seconds
We run only two computations, so we see two dots. let’s run the full monty:
$ py.test -q --all
collecting ... collected 5 items
....F
================================= FAILURES =================================
_____________________________ test_compute[4] ______________________________
param1 = 4
def test_compute(param1):
> assert param1 < 4
E assert 4 < 4
test_compute.py:3: AssertionError
1 failed, 4 passed in 0.01 seconds
As expected when running the full range of param1 values we’ll get an error on the last one.
The parametrization of test functions happens at collection time. It is often a good idea to setup possibly expensive resources only when the actual test is run. Here is a simple example how you can achieve that:
# content of test_backends.py
import pytest
def test_db_initialized(db):
# a dummy test
if db.__class__.__name__ == "DB2":
pytest.fail("deliberately failing for demo purposes")
Now we add a test configuration that takes care to generate two invocations of the test_db_initialized function and furthermore a factory that creates a database object when each test is actually run:
# content of conftest.py
def pytest_generate_tests(metafunc):
if 'db' in metafunc.funcargnames:
metafunc.addcall(param="d1")
metafunc.addcall(param="d2")
class DB1:
"one database object"
class DB2:
"alternative database object"
def pytest_funcarg__db(request):
if request.param == "d1":
return DB1()
elif request.param == "d2":
return DB2()
else:
raise ValueError("invalid internal test config")
Let’s first see how it looks like at collection time:
$ py.test test_backends.py --collectonly
=========================== test session starts ============================
platform linux2 -- Python 2.7.1 -- pytest-2.1.1
collecting ... collected 2 items
<Module 'test_backends.py'>
<Function 'test_db_initialized[0]'>
<Function 'test_db_initialized[1]'>
============================= in 0.00 seconds =============================
And then when we run the test:
$ py.test -q test_backends.py
collecting ... collected 2 items
.F
================================= FAILURES =================================
__________________________ test_db_initialized[1] __________________________
db = <conftest.DB2 instance at 0x17829e0>
def test_db_initialized(db):
# a dummy test
if db.__class__.__name__ == "DB2":
> pytest.fail("deliberately failing for demo purposes")
E Failed: deliberately failing for demo purposes
test_backends.py:6: Failed
1 failed, 1 passed in 0.01 seconds
Now you see that one invocation of the test passes and another fails, as it to be expected.
Here is an example pytest_generate_function function implementing a parametrization scheme similar to Michael Foords unittest parameterizer in a lot less code:
# content of ./test_parametrize.py
import pytest
def pytest_generate_tests(metafunc):
# called once per each test function
for funcargs in metafunc.cls.params[metafunc.function.__name__]:
# schedule a new test function run with applied **funcargs
metafunc.addcall(funcargs=funcargs)
class TestClass:
# a map specifying multiple argument sets for a test method
params = {
'test_equals': [dict(a=1, b=2), dict(a=3, b=3), ],
'test_zerodivision': [dict(a=1, b=0), dict(a=3, b=2)],
}
def test_equals(self, a, b):
assert a == b
def test_zerodivision(self, a, b):
pytest.raises(ZeroDivisionError, "a/b")
Running it means we are two tests for each test functions, using the respective settings:
$ py.test -q
collecting ... collected 6 items
.FF..F
================================= FAILURES =================================
__________________________ test_db_initialized[1] __________________________
db = <conftest.DB2 instance at 0x2acf4d0>
def test_db_initialized(db):
# a dummy test
if db.__class__.__name__ == "DB2":
> pytest.fail("deliberately failing for demo purposes")
E Failed: deliberately failing for demo purposes
test_backends.py:6: Failed
_________________________ TestClass.test_equals[0] _________________________
self = <test_parametrize.TestClass instance at 0x2ad2830>, a = 1, b = 2
def test_equals(self, a, b):
> assert a == b
E assert 1 == 2
test_parametrize.py:17: AssertionError
______________________ TestClass.test_zerodivision[1] ______________________
self = <test_parametrize.TestClass instance at 0x2ad8830>, a = 3, b = 2
def test_zerodivision(self, a, b):
> pytest.raises(ZeroDivisionError, "a/b")
E Failed: DID NOT RAISE
test_parametrize.py:20: Failed
3 failed, 3 passed in 0.02 seconds
Modifying the previous example we can also allow decorators for parametrizing test methods:
# content of test_parametrize2.py
import pytest
# test support code
def params(funcarglist):
def wrapper(function):
function.funcarglist = funcarglist
return function
return wrapper
def pytest_generate_tests(metafunc):
for funcargs in getattr(metafunc.function, 'funcarglist', ()):
metafunc.addcall(funcargs=funcargs)
# actual test code
class TestClass:
@params([dict(a=1, b=2), dict(a=3, b=3), ])
def test_equals(self, a, b):
assert a == b
@params([dict(a=1, b=0), dict(a=3, b=2)])
def test_zerodivision(self, a, b):
pytest.raises(ZeroDivisionError, "a/b")
Running it gives similar results as before:
$ py.test -q test_parametrize2.py
collecting ... collected 4 items
F..F
================================= FAILURES =================================
_________________________ TestClass.test_equals[0] _________________________
self = <test_parametrize2.TestClass instance at 0x1ef2170>, a = 1, b = 2
@params([dict(a=1, b=2), dict(a=3, b=3), ])
def test_equals(self, a, b):
> assert a == b
E assert 1 == 2
test_parametrize2.py:19: AssertionError
______________________ TestClass.test_zerodivision[1] ______________________
self = <test_parametrize2.TestClass instance at 0x20e4248>, a = 3, b = 2
@params([dict(a=1, b=0), dict(a=3, b=2)])
def test_zerodivision(self, a, b):
> pytest.raises(ZeroDivisionError, "a/b")
E Failed: DID NOT RAISE
test_parametrize2.py:23: Failed
2 failed, 2 passed in 0.02 seconds
Here is a stripped down real-life example of using parametrized testing for testing serialization between different interpreters. We define a test_basic_objects function which is to be run with different sets of arguments for its three arguments:
* ``python1``: first python interpreter
* ``python2``: second python interpreter
* ``obj``: object to be dumped from first interpreter and loaded into second interpreter
"""
module containing a parametrized tests testing cross-python
serialization via the pickle module.
"""
import py
pythonlist = ['python2.4', 'python2.5', 'python2.6', 'python2.7', 'python2.8']
def pytest_generate_tests(metafunc):
if 'python1' in metafunc.funcargnames:
assert 'python2' in metafunc.funcargnames
for obj in metafunc.function.multiarg.kwargs['obj']:
for py1 in pythonlist:
for py2 in pythonlist:
metafunc.addcall(id="%s-%s-%s" % (py1, py2, obj),
param=(py1, py2, obj))
@py.test.mark.multiarg(obj=[42, {}, {1:3},])
def test_basic_objects(python1, python2, obj):
python1.dumps(obj)
python2.load_and_is_true("obj == %s" % obj)
def pytest_funcarg__python1(request):
tmpdir = request.getfuncargvalue("tmpdir")
picklefile = tmpdir.join("data.pickle")
return Python(request.param[0], picklefile)
def pytest_funcarg__python2(request):
python1 = request.getfuncargvalue("python1")
return Python(request.param[1], python1.picklefile)
def pytest_funcarg__obj(request):
return request.param[2]
class Python:
def __init__(self, version, picklefile):
self.pythonpath = py.path.local.sysfind(version)
if not self.pythonpath:
py.test.skip("%r not found" %(version,))
self.picklefile = picklefile
def dumps(self, obj):
dumpfile = self.picklefile.dirpath("dump.py")
dumpfile.write(py.code.Source("""
import pickle
f = open(%r, 'wb')
s = pickle.dump(%r, f)
f.close()
""" % (str(self.picklefile), obj)))
py.process.cmdexec("%s %s" %(self.pythonpath, dumpfile))
def load_and_is_true(self, expression):
loadfile = self.picklefile.dirpath("load.py")
loadfile.write(py.code.Source("""
import pickle
f = open(%r, 'rb')
obj = pickle.load(f)
f.close()
res = eval(%r)
if not res:
raise SystemExit(1)
""" % (str(self.picklefile), expression)))
print (loadfile)
py.process.cmdexec("%s %s" %(self.pythonpath, loadfile))
Running it (with Python-2.4 through to Python2.7 installed):
. $ py.test -q multipython.py
collecting ... collected 75 items
....s....s....s....ssssss....s....s....s....ssssss....s....s....s....ssssss
48 passed, 27 skipped in 2.48 seconds