diff --git a/pythonFiles/testing_tools/__init__.py b/pythonFiles/testing_tools/__init__.py index e69de29bb2d1..5b7f7a925cc0 100644 --- a/pythonFiles/testing_tools/__init__.py +++ b/pythonFiles/testing_tools/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/testing_tools/adapter/__init__.py b/pythonFiles/testing_tools/adapter/__init__.py index e69de29bb2d1..5b7f7a925cc0 100644 --- a/pythonFiles/testing_tools/adapter/__init__.py +++ b/pythonFiles/testing_tools/adapter/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/testing_tools/adapter/__main__.py b/pythonFiles/testing_tools/adapter/__main__.py index 9aa10f0526e9..59908ce641d4 100644 --- a/pythonFiles/testing_tools/adapter/__main__.py +++ b/pythonFiles/testing_tools/adapter/__main__.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import absolute_import import argparse @@ -7,10 +10,6 @@ from .errors import UnsupportedToolError, UnsupportedCommandError -# Set this to True to pretty-print the output. -DEBUG=False -#DEBUG=True - TOOLS = { 'pytest': { '_add_subparser': pytest.add_cli_subparser, @@ -22,6 +21,7 @@ } + def parse_args( argv=sys.argv[1:], prog=sys.argv[0], @@ -40,6 +40,9 @@ def parse_args( # Add "run" and "debug" subcommands when ready. for cmdname in ['discover']: sub = cmdsubs.add_parser(cmdname) + if cmdname == 'discover': + sub.add_argument('--simple', action='store_true') + sub.add_argument('--show-pytest', action='store_true') subsubs = sub.add_subparsers(dest='tool') for toolname in sorted(TOOLS): try: @@ -55,6 +58,14 @@ def parse_args( cmd = ns.pop('cmd') if not cmd: parser.error('missing command') + if cmd == 'discover': + if '--simple' in toolargs: + toolargs.remove('--simple') + ns['simple'] = True + if '--show-pytest' in toolargs: + toolargs.remove('--show-pytest') + ns['show_pytest'] = True + tool = ns.pop('tool') if not tool: parser.error('missing tool') @@ -75,8 +86,11 @@ def main(toolname, cmdname, subargs, toolargs, except KeyError: raise UnsupportedCommandError(cmdname) - result = run(toolargs, **subargs) - report_result(result, debug=DEBUG) + parents, result = run(toolargs, **subargs) + report_result(result, parents, + debug=('-v' in toolargs or '--verbose' in toolargs), + **subargs + ) if __name__ == '__main__': diff --git a/pythonFiles/testing_tools/adapter/errors.py b/pythonFiles/testing_tools/adapter/errors.py index d0f8563ba15c..18b3819dcbdb 100644 --- a/pythonFiles/testing_tools/adapter/errors.py +++ b/pythonFiles/testing_tools/adapter/errors.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + class UnsupportedToolError(ValueError): def __init__(self, tool): diff --git a/pythonFiles/testing_tools/adapter/info.py b/pythonFiles/testing_tools/adapter/info.py index cd02822503ab..c9c14571dd6b 100644 --- a/pythonFiles/testing_tools/adapter/info.py +++ b/pythonFiles/testing_tools/adapter/info.py @@ -1,9 +1,114 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from collections import namedtuple class TestPath(namedtuple('TestPath', 'root relfile func sub')): """Where to find a single test.""" + def __new__(cls, root, relfile, func, sub=None): + self = super(TestPath, cls).__new__( + cls, + str(root) if root else None, + str(relfile) if relfile else None, + str(func) if func else None, + [str(s) for s in sub] if sub else None, + ) + return self + + def __init__(self, *args, **kwargs): + if self.root is None: + raise TypeError('missing id') + if self.relfile is None: + raise TypeError('missing kind') + # self.func may be None (e.g. for doctests). + # self.sub may be None. + + +class ParentInfo(namedtuple('ParentInfo', 'id kind name root parentid')): + + KINDS = ('folder', 'file', 'suite', 'function', 'subtest') + + def __new__(cls, id, kind, name, root=None, parentid=None): + self = super(ParentInfo, cls).__new__( + cls, + str(id) if id else None, + str(kind) if kind else None, + str(name) if name else None, + str(root) if root else None, + str(parentid) if parentid else None, + ) + return self + + def __init__(self, *args, **kwargs): + if self.id is None: + raise TypeError('missing id') + if self.kind is None: + raise TypeError('missing kind') + if self.kind not in self.KINDS: + raise ValueError('unsupported kind {!r}'.format(self.kind)) + if self.name is None: + raise TypeError('missing name') + if self.root is None: + if self.parentid is not None or self.kind != 'folder': + raise TypeError('missing root') + elif self.parentid is None: + raise TypeError('missing parentid') + -class TestInfo(namedtuple('TestInfo', 'id name path lineno markers')): +class TestInfo(namedtuple('TestInfo', 'id name path source markers parentid kind')): """Info for a single test.""" + + MARKERS = ('skip', 'skip-if', 'expected-failure') + KINDS = ('function', 'doctest') + + def __new__(cls, id, name, path, source, markers, parentid, kind='function'): + self = super(TestInfo, cls).__new__( + cls, + str(id) if id else None, + str(name) if name else None, + path or None, + str(source) if source else None, + [str(marker) for marker in markers or ()], + str(parentid) if parentid else None, + str(kind) if kind else None, + ) + return self + + def __init__(self, *args, **kwargs): + if self.id is None: + raise TypeError('missing id') + if self.name is None: + raise TypeError('missing name') + if self.path is None: + raise TypeError('missing path') + if self.source is None: + raise TypeError('missing source') + else: + srcfile, _, lineno = self.source.rpartition(':') + if not srcfile or not lineno or int(lineno) < 0: + raise ValueError('bad source {!r}'.format(self.source)) + if self.markers: + badmarkers = [m for m in self.markers if m not in self.MARKERS] + if badmarkers: + raise ValueError('unsupported markers {!r}'.format(badmarkers)) + if self.parentid is None: + raise TypeError('missing parentid') + if self.kind is None: + raise TypeError('missing kind') + elif self.kind not in self.KINDS: + raise ValueError('unsupported kind {!r}'.format(self.kind)) + + + @property + def root(self): + return self.path.root + + @property + def srcfile(self): + return self.source.rpartition(':')[0] + + @property + def lineno(self): + return int(self.source.rpartition(':')[-1]) diff --git a/pythonFiles/testing_tools/adapter/pytest.py b/pythonFiles/testing_tools/adapter/pytest.py index efc499f5a979..8dd1e25eb4af 100644 --- a/pythonFiles/testing_tools/adapter/pytest.py +++ b/pythonFiles/testing_tools/adapter/pytest.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import absolute_import import os.path @@ -5,7 +8,7 @@ import pytest from .errors import UnsupportedCommandError -from .info import TestInfo, TestPath +from .info import TestInfo, TestPath, ParentInfo def add_cli_subparser(cmd, name, parent): @@ -19,26 +22,35 @@ def add_cli_subparser(cmd, name, parent): return parser -def discover(pytestargs=None, - _pytest_main=pytest.main, _plugin=None): +def discover(pytestargs=None, show_pytest=False, + _pytest_main=pytest.main, _plugin=None, **kwargs): """Return the results of test discovery.""" if _plugin is None: _plugin = TestCollector() - pytestargs = _adjust_pytest_args(pytestargs) + pytestargs = _adjust_pytest_args(pytestargs, show_pytest=show_pytest) ec = _pytest_main(pytestargs, [_plugin]) if ec != 0: raise Exception('pytest discovery failed (exit code {})'.format(ec)) - if _plugin.discovered is None: + if not _plugin._started: raise Exception('pytest discovery did not start') - return _plugin.discovered + return ( + _plugin._tests.parents, + #[p._replace( + # id=p.id.lstrip('.' + os.path.sep), + # parentid=p.parentid.lstrip('.' + os.path.sep), + # ) + # for p in _plugin._tests.parents], + list(_plugin._tests), + ) -def _adjust_pytest_args(pytestargs): +def _adjust_pytest_args(pytestargs, show_pytest): pytestargs = list(pytestargs) if pytestargs else [] # Duplicate entries should be okay. pytestargs.insert(0, '--collect-only') - pytestargs.insert(0, '-pno:terminal') + if not show_pytest: + pytestargs.insert(0, '-pno:terminal') # TODO: pull in code from: # src/client/unittests/pytest/services/discoveryService.ts # src/client/unittests/pytest/services/argsService.ts @@ -48,37 +60,141 @@ def _adjust_pytest_args(pytestargs): class TestCollector(object): """This is a pytest plugin that collects the discovered tests.""" - discovered = None + NORMCASE = staticmethod(os.path.normcase) + PATHSEP = os.path.sep + + def __init__(self, tests=None): + if tests is None: + tests = DiscoveredTests() + self._tests = tests + self._started = False # Relevant plugin hooks: # https://docs.pytest.org/en/latest/reference.html#collection-hooks def pytest_collection_modifyitems(self, session, config, items): - self.discovered = [] + self._started = True + self._tests.reset() for item in items: - info = _parse_item(item) - self.discovered.append(info) + test, suiteids = _parse_item(item, self.NORMCASE, self.PATHSEP) + self._tests.add_test(test, suiteids) # This hook is not specified in the docs, so we also provide # the "modifyitems" hook just in case. def pytest_collection_finish(self, session): + self._started = True try: items = session.items except AttributeError: # TODO: Is there an alternative? return -# print(', '.join(k for k in dir(items[0]) if k[0].islower())) - self.discovered = [] + self._tests.reset() for item in items: -# print(' ', item.user_properties) -# print(' ', item.own_markers) -# print(' ', list(item.iter_markers())) -# print() - info = _parse_item(item) - self.discovered.append(info) + test, suiteids = _parse_item(item, self.NORMCASE, self.PATHSEP) + self._tests.add_test(test, suiteids) + + +class DiscoveredTests(object): + + def __init__(self): + self.reset() + + def __len__(self): + return len(self._tests) + + def __getitem__(self, index): + return self._tests[index] + + @property + def parents(self): + return sorted(self._parents.values(), key=lambda v: (v.root or v.name, v.id)) + + def reset(self): + self._parents = {} + self._tests = [] + + def add_test(self, test, suiteids): + parentid = self._ensure_parent(test.path, test.parentid, suiteids) + test = test._replace(parentid=parentid) + if not test.id.startswith('.' + os.path.sep): + test = test._replace(id=os.path.join('.', test.id)) + self._tests.append(test) + + def _ensure_parent(self, path, parentid, suiteids): + if not parentid.startswith('.' + os.path.sep): + parentid = os.path.join('.', parentid) + fileid = self._ensure_file(path.root, path.relfile) + rootdir = path.root + + if not path.func: + return parentid + + fullsuite, _, funcname = path.func.rpartition('.') + suiteid = self._ensure_suites(fullsuite, rootdir, fileid, suiteids) + parent = suiteid if suiteid else fileid + + if path.sub: + if (rootdir, parentid) not in self._parents: + funcinfo = ParentInfo(parentid, 'function', funcname, + rootdir, parent) + self._parents[(rootdir, parentid)] = funcinfo + elif parent != parentid: + # TODO: What to do? + raise NotImplementedError + return parentid + + def _ensure_file(self, rootdir, relfile): + if (rootdir, '.') not in self._parents: + self._parents[(rootdir, '.')] = ParentInfo('.', 'folder', rootdir) + if relfile.startswith('.' + os.path.sep): + fileid = relfile + else: + fileid = relfile = os.path.join('.', relfile) + + if (rootdir, fileid) not in self._parents: + folderid, filebase = os.path.split(fileid) + fileinfo = ParentInfo(fileid, 'file', filebase, rootdir, folderid) + self._parents[(rootdir, fileid)] = fileinfo + + while folderid != '.' and (rootdir, folderid) not in self._parents: + parentid, name = os.path.split(folderid) + folderinfo = ParentInfo(folderid, 'folder', name, rootdir, parentid) + self._parents[(rootdir, folderid)] = folderinfo + folderid = parentid + return relfile + + def _ensure_suites(self, fullsuite, rootdir, fileid, suiteids): + if not fullsuite: + if suiteids: + # TODO: What to do? + raise NotImplementedError + return None + if len(suiteids) != fullsuite.count('.') + 1: + # TODO: What to do? + raise NotImplementedError + + suiteid = suiteids.pop() + if not suiteid.startswith('.' + os.path.sep): + suiteid = os.path.join('.', suiteid) + final = suiteid + while '.' in fullsuite and (rootdir, suiteid) not in self._parents: + parentid = suiteids.pop() + if not parentid.startswith('.' + os.path.sep): + parentid = os.path.join('.', parentid) + fullsuite, _, name = fullsuite.rpartition('.') + suiteinfo = ParentInfo(suiteid, 'suite', name, rootdir, parentid) + self._parents[(rootdir, suiteid)] = suiteinfo + suiteid = parentid + else: + name = fullsuite + suiteinfo = ParentInfo(suiteid, 'suite', name, rootdir, fileid) + if (rootdir, suiteid) not in self._parents: + self._parents[(rootdir, suiteid)] = suiteinfo + return final -def _parse_item(item): + +def _parse_item(item, _normcase, _pathsep): """ (pytest.Collector) pytest.Session @@ -89,39 +205,62 @@ def _parse_item(item): (pytest.Item) pytest.Function """ + #_debug_item(item, showsummary=True) + kind, _ = _get_item_kind(item) + # Figure out the func, suites, and subs. + (fileid, suiteids, suites, funcid, basename, parameterized + ) = _parse_node_id(item.nodeid, kind) + if kind == 'function': + funcname = basename + if funcid and item.function.__name__ != funcname: + # TODO: What to do? + raise NotImplementedError + if suites: + testfunc = '.'.join(suites) + '.' + funcname + else: + testfunc = funcname + elif kind == 'doctest': + testfunc = None + funcname = None + # Figure out the file. - filename, lineno, fullname = item.location - if not str(item.fspath).endswith(os.path.sep + filename): + fspath = str(item.fspath) + if not fspath.endswith(_pathsep + fileid): raise NotImplementedError - testroot = str(item.fspath)[:-len(filename)].rstrip(os.path.sep) - if os.path.sep in filename: + filename = fspath[-len(fileid):] + testroot = str(item.fspath)[:-len(fileid)].rstrip(_pathsep) + if _pathsep in filename: relfile = filename else: - relfile = os.path.join('.', filename) - - # Figure out the func (and subs). - funcname = item.function.__name__ - parts = item.nodeid.split('::') - if parts.pop(0) != filename: - # TODO: What to do? - raise NotImplementedError - suites = [] - while len(parts) > 1: - suites.append(parts.pop(0)) - parameterized = '' - if '[' in parts[0]: - _func, sep, parameterized = parts[0].partition('[') - parameterized = sep + parameterized - if _func != funcname: + relfile = '.' + _pathsep + filename + srcfile, lineno, fullname = item.location + if srcfile != fileid: + # pytest supports discovery of tests imported from other + # modules. This is reflected by a different filename + # in item.location. + if _normcase(fileid) == _normcase(srcfile): + srcfile = fileid + else: + srcfile = relfile + location = '{}:{}'.format(srcfile, lineno) + if kind == 'function': + if testfunc and fullname != testfunc + parameterized: + print(fullname, testfunc) # TODO: What to do? raise NotImplementedError - if suites: - testfunc = '.'.join(suites) + '.' + funcname + elif kind == 'doctest': + if testfunc and fullname != testfunc + parameterized: + print(fullname, testfunc) + # TODO: What to do? + raise NotImplementedError + + # Sort out the parent. + if parameterized: + parentid = funcid + elif suites: + parentid = suiteids[-1] else: - testfunc = funcname - if fullname != testfunc + parameterized: - # TODO: What to do? - raise NotImplementedError + parentid = fileid # Sort out markers. # See: https://docs.pytest.org/en/latest/reference.html#marks @@ -138,7 +277,7 @@ def _parse_item(item): markers.add('expected-failure') # TODO: Support other markers? - return TestInfo( + test = TestInfo( id=item.nodeid, name=item.name, path=TestPath( @@ -147,6 +286,116 @@ def _parse_item(item): func=testfunc, sub=[parameterized] if parameterized else None, ), - lineno=lineno, + source=location, markers=sorted(markers) if markers else None, + parentid=parentid, ) + return test, suiteids + + +def _parse_node_id(nodeid, kind='function'): + if kind == 'doctest': + try: + parentid, name = nodeid.split('::') + except ValueError: + # TODO: Unexpected! What to do? + raise NotImplementedError + funcid = None + parameterized = '' + else: + parameterized = '' + if nodeid.endswith(']'): + funcid, sep, parameterized = nodeid.partition('[') + if not sep: + # TODO: Unexpected! What to do? + raise NotImplementedError + parameterized = sep + parameterized + else: + funcid = nodeid + + parentid, _, name = funcid.rpartition('::') + if not name: + # TODO: What to do? We expect at least a filename and a function + raise NotImplementedError + + suites = [] + suiteids = [] + while '::' in parentid: + suiteids.insert(0, parentid) + parentid, _, suitename = parentid.rpartition('::') + suites.insert(0, suitename) + fileid = parentid + + return fileid, suiteids, suites, funcid, name, parameterized + + +def _get_item_kind(item): + """Return (kind, isunittest) for the given item.""" + try: + itemtype = item.kind + except AttributeError: + itemtype = item.__class__.__name__ + + if itemtype == 'DoctestItem': + return 'doctest', False + elif itemtype == 'Function': + return 'function', False + elif itemtype == 'TestCaseFunction': + return 'function', True + elif item.hasattr('function'): + return 'function', False + else: + return None, False + + +############################# +# useful for debugging + +def _debug_item(item, showsummary=False): + item._debugging = True + try: + # TODO: Make a PytestTest class to wrap the item? + summary = { + 'id': item.nodeid, + 'kind': _get_item_kind(item), + 'class': item.__class__.__name__, + 'name': item.name, + 'fspath': item.fspath, + 'location': item.location, + 'func': getattr(item, 'function', None), + 'markers': item.own_markers, + #'markers': list(item.iter_markers()), + 'props': item.user_properties, + 'attrnames': dir(item), + } + finally: + item._debugging = False + + if showsummary: + print(item.nodeid) + for key in ('kind', 'class', 'name', 'fspath', 'location', 'func', + 'markers', 'props'): + print(' {:12} {}'.format(key, summary[key])) + print() + + return summary + + +def _group_attr_names(attrnames): + grouped = { + 'dunder': [n for n in attrnames + if n.startswith('__') and n.endswith('__')], + 'private': [n for n in attrnames if n.startswith('_')], + 'constants': [n for n in attrnames if n.isupper()], + 'classes': [n for n in attrnames + if n == n.capitalize() and not n.isupper()], + 'vars': [n for n in attrnames if n.islower()], + } + grouped['other'] = [n for n in attrnames + if n not in grouped['dunder'] + and n not in grouped['private'] + and n not in grouped['constants'] + and n not in grouped['classes'] + and n not in grouped['vars'] + ] + return grouped diff --git a/pythonFiles/testing_tools/adapter/report.py b/pythonFiles/testing_tools/adapter/report.py index b3f72343b708..6d8df63a0ed0 100644 --- a/pythonFiles/testing_tools/adapter/report.py +++ b/pythonFiles/testing_tools/adapter/report.py @@ -1,12 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + from __future__ import print_function import json -def report_discovered(tests, debug=False, - _send=print): +def report_discovered(tests, parents, debug=False, simple=False, + _send=print, **kwargs): """Serialize the discovered tests and write to stdout.""" - data = [{ + if simple: + data = [{ 'id': test.id, 'name': test.name, 'testroot': test.path.root, @@ -14,8 +18,49 @@ def report_discovered(tests, debug=False, 'lineno': test.lineno, 'testfunc': test.path.func, 'subtest': test.path.sub or None, - 'markers': test.markers or None, + 'markers': test.markers or [], } for test in tests] + else: + byroot = {} + for parent in parents: + rootdir = parent.name if parent.root is None else parent.root + try: + root = byroot[rootdir] + except KeyError: + root = byroot[rootdir] = { + 'id': rootdir, + 'parents': [], + 'tests': [], + } + if not parent.root: + root['id'] = parent.id + continue + root['parents'].append({ + 'id': parent.id, + 'kind': parent.kind, + 'name': parent.name, + 'parentid': parent.parentid, + }) + for test in tests: + # We are guaranteed that the parent was added. + root = byroot[test.path.root] + testdata = { + 'id': test.id, + 'name': test.name, + # TODO: Add a "kind" field + # (e.g. "unittest", "function", "doctest") + 'source': test.source, + 'markers': test.markers or [], + 'parentid': test.parentid, + } + root['tests'].append(testdata) + data = [{ + 'rootid': byroot[root]['id'], + 'root': root, + 'parents': byroot[root]['parents'], + 'tests': byroot[root]['tests'], + } for root in sorted(byroot)] + kwargs = {} if debug: # human-formatted @@ -25,4 +70,5 @@ def report_discovered(tests, debug=False, separators=(',', ': '), ) serialized = json.dumps(data, **kwargs) + _send(serialized) diff --git a/pythonFiles/testing_tools/run_adapter.py b/pythonFiles/testing_tools/run_adapter.py index 64b58ed62822..d21f3efd1b70 100644 --- a/pythonFiles/testing_tools/run_adapter.py +++ b/pythonFiles/testing_tools/run_adapter.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + # Replace the "." entry. import os.path import sys diff --git a/pythonFiles/tests/datascience/__init__.py b/pythonFiles/tests/datascience/__init__.py index e69de29bb2d1..5b7f7a925cc0 100644 --- a/pythonFiles/tests/datascience/__init__.py +++ b/pythonFiles/tests/datascience/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/tests/testing_tools/adapter/test___main__.py b/pythonFiles/tests/testing_tools/adapter/test___main__.py index b5d743e31101..d0e8247d3021 100644 --- a/pythonFiles/tests/testing_tools/adapter/test___main__.py +++ b/pythonFiles/tests/testing_tools/adapter/test___main__.py @@ -27,8 +27,8 @@ class StubReporter(StubProxy): def __init__(self, stub=None): super(StubReporter, self).__init__(stub, 'reporter') - def report(self, discovered, **kwargs): - self.add_call('report', (discovered,), kwargs or None) + def report(self, tests, parents, **kwargs): + self.add_call('report', (tests, parents), kwargs or None) ################################## @@ -55,7 +55,7 @@ def test_pytest_default(self): self.assertEqual(tool, 'pytest') self.assertEqual(cmd, 'discover') - self.assertEqual(args, {}) + self.assertEqual(args, {'show_pytest': False, 'simple': False}) self.assertEqual(toolargs, []) def test_pytest_full(self): @@ -73,7 +73,7 @@ def test_pytest_full(self): self.assertEqual(tool, 'pytest') self.assertEqual(cmd, 'discover') - self.assertEqual(args, {}) + self.assertEqual(args, {'show_pytest': False, 'simple': False}) self.assertEqual(toolargs, [ '--', '--strict', @@ -99,8 +99,8 @@ class MainTests(unittest.TestCase): def test_discover(self): stub = Stub() tool = StubTool('spamspamspam', stub) - expected = object() - tool.return_discover = expected + tests, parents = object(), object() + tool.return_discover = (parents, tests) reporter = StubReporter(stub) main(tool.name, 'discover', {'spam': 'eggs'}, [], _tools={tool.name: { @@ -112,7 +112,8 @@ def test_discover(self): self.assertEqual(tool.calls, [ ('spamspamspam.discover', ([],), {'spam': 'eggs'}), - ('reporter.report', (expected,), {'debug': False}), + ('reporter.report', (tests, parents), {'debug': False, + 'spam': 'eggs'}), ]) def test_unsupported_tool(self): diff --git a/pythonFiles/tests/testing_tools/adapter/test_pytest.py b/pythonFiles/tests/testing_tools/adapter/test_pytest.py index 2602d9d05af0..764f323564a3 100644 --- a/pythonFiles/tests/testing_tools/adapter/test_pytest.py +++ b/pythonFiles/tests/testing_tools/adapter/test_pytest.py @@ -1,14 +1,15 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os import os.path import unittest from ...util import Stub, StubProxy from testing_tools.adapter.errors import UnsupportedCommandError -from testing_tools.adapter.info import TestInfo, TestPath +from testing_tools.adapter.info import TestInfo, TestPath, ParentInfo from testing_tools.adapter.pytest import ( - discover, add_cli_subparser, TestCollector + discover, add_cli_subparser, TestCollector, DiscoveredTests ) @@ -44,8 +45,13 @@ def main(self, args, plugins): class StubPlugin(StubProxy): - def __init__(self, stub=None): + _started = True + + def __init__(self, stub=None, tests=None): super(StubPlugin, self).__init__(stub, 'plugin') + if tests is None: + tests = StubDiscoveredTests(self.stub) + self._tests = tests def __getattr__(self, name): if not name.startswith('pytest_'): @@ -55,6 +61,35 @@ def func(*args, **kwargs): return func +class StubDiscoveredTests(StubProxy): + + NOT_FOUND = object() + + def __init__(self, stub=None): + super(StubDiscoveredTests, self).__init__(stub, 'discovered') + self.return_items = [] + self.return_parents = [] + + def __len__(self): + self.add_call('__len__', None, None) + return len(self.return_items) + + def __getitem__(self, index): + self.add_call('__getitem__', (index,), None) + return self.return_items[index] + + @property + def parents(self): + self.add_call('parents', None, None) + return self.return_parents + + def reset(self): + self.add_call('reset', None, None) + + def add_test(self, test, suiteids): + self.add_call('add_test', None, {'test': test, 'suiteids': suiteids}) + + class FakeFunc(object): def __init__(self, name): @@ -69,14 +104,30 @@ def __init__(self, name): class StubPytestItem(StubProxy): + kind = 'Function' + + _debugging = False + _hasfunc = True + def __init__(self, stub=None, **attrs): super(StubPytestItem, self).__init__(stub, 'pytest.Item') + if attrs.get('function') is None: + attrs.pop('function', None) + self._hasfunc = False + + attrs.setdefault('user_properties', []) + self.__dict__.update(attrs) + if 'own_markers' not in attrs: self.own_markers = () def __getattr__(self, name): - self.add_call(name + ' (attr)', None, None) + if not self._debugging: + self.add_call(name + ' (attr)', None, None) + if name == 'function': + if not self._hasfunc: + raise AttributeError(name) def func(*args, **kwargs): self.add_call(name, args or None, kwargs or None) return func @@ -149,17 +200,21 @@ class DiscoverTests(unittest.TestCase): def test_basic(self): stub = Stub() - pytest = StubPyTest(stub) + stubpytest = StubPyTest(stub) plugin = StubPlugin(stub) expected = [] plugin.discovered = expected - discovered = discover([], _pytest_main=pytest.main, _plugin=plugin) + parents, tests = discover([], _pytest_main=stubpytest.main, _plugin=plugin) - self.assertEqual(discovered, expected) + self.assertEqual(parents, []) + self.assertEqual(tests, expected) self.assertEqual(stub.calls, [ ('pytest.main', None, {'args': self.DEFAULT_ARGS, 'plugins': [plugin]}), + ('discovered.parents', None, None), + ('discovered.__len__', None, None), + ('discovered.__getitem__', (0,), None), ]) def test_failure(self): @@ -181,9 +236,10 @@ class CollectorTests(unittest.TestCase): def test_modifyitems(self): stub = Stub() + discovered = StubDiscoveredTests(stub) session = StubPytestSession(stub) config = StubPytestConfig(stub) - collector = TestCollector() + collector = TestCollector(tests=discovered) testroot = '/a/b/c'.replace('/', os.path.sep) relfile1 = './test_spam.py'.replace('/', os.path.sep) @@ -214,6 +270,14 @@ def test_modifyitems(self): fspath=os.path.join(testroot, 'test_spam.py'), function=FakeFunc('test_all'), ), + StubPytestItem( + stub, + nodeid='test_spam.py::test_each[10-10]', + name='test_each[10-10]', + location=('test_spam.py', 273, 'test_each[10-10]'), + fspath=os.path.join(testroot, 'test_spam.py'), + function=FakeFunc('test_each'), + ), StubPytestItem( stub, nodeid=relfile2 + '::All::BasicTests::test_first', @@ -243,102 +307,964 @@ def test_modifyitems(self): ]) self.maxDiff = None - self.assertEqual(collector.discovered, [ + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=['test_spam.py::SpamTests'], + test=TestInfo( + id='test_spam.py::SpamTests::test_one', + name='test_one', + path=TestPath( + root=testroot, + relfile=relfile1, + func='SpamTests.test_one', + sub=None, + ), + source='{}:{}'.format(relfile1, 12), + markers=None, + parentid='test_spam.py::SpamTests', + ), + )), + ('discovered.add_test', None, dict( + suiteids=['test_spam.py::SpamTests'], + test=TestInfo( + id='test_spam.py::SpamTests::test_other', + name='test_other', + path=TestPath( + root=testroot, + relfile=relfile1, + func='SpamTests.test_other', + sub=None, + ), + source='{}:{}'.format(relfile1, 19), + markers=None, + parentid='test_spam.py::SpamTests', + ), + )), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id='test_spam.py::test_all', + name='test_all', + path=TestPath( + root=testroot, + relfile=relfile1, + func='test_all', + sub=None, + ), + source='{}:{}'.format(relfile1, 144), + markers=None, + parentid='test_spam.py', + ), + )), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id='test_spam.py::test_each[10-10]', + name='test_each[10-10]', + path=TestPath( + root=testroot, + relfile=relfile1, + func='test_each', + sub=['[10-10]'], + ), + source='{}:{}'.format(relfile1, 273), + markers=None, + parentid='test_spam.py::test_each', + ), + )), + ('discovered.add_test', None, dict( + suiteids=[relfile2 + '::All', + relfile2 + '::All::BasicTests'], + test=TestInfo( + id=relfile2 + '::All::BasicTests::test_first', + name='test_first', + path=TestPath( + root=testroot, + relfile=relfile2, + func='All.BasicTests.test_first', + sub=None, + ), + source='{}:{}'.format(relfile2, 31), + markers=None, + parentid=relfile2 + '::All::BasicTests', + ), + )), + ('discovered.add_test', None, dict( + suiteids=[relfile2 + '::All', + relfile2 + '::All::BasicTests'], + test=TestInfo( + id=relfile2 + '::All::BasicTests::test_each[1+2-3]', + name='test_each[1+2-3]', + path=TestPath( + root=testroot, + relfile=relfile2, + func='All.BasicTests.test_each', + sub=['[1+2-3]'], + ), + source='{}:{}'.format(relfile2, 62), + markers=['expected-failure', 'skip', 'skip-if'], + parentid=relfile2 + '::All::BasicTests::test_each', + ), + )), + ]) + + def test_finish(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) + session.items = [ + StubPytestItem( + stub, + nodeid=relfile + '::SpamTests::test_spam', + name='test_spam', + location=(relfile, 12, 'SpamTests.test_spam'), + fspath=os.path.join(testroot, relfile), + function=FakeFunc('test_spam'), + ), + ] + collector = TestCollector(tests=discovered) + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[relfile + '::SpamTests'], + test=TestInfo( + id=relfile + '::SpamTests::test_spam', + name='test_spam', + path=TestPath( + root=testroot, + relfile=relfile, + func='SpamTests.test_spam', + sub=None, + ), + source='{}:{}'.format(relfile, 12), + markers=None, + parentid=relfile + '::SpamTests', + ), + )), + ]) + + def test_doctest(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + doctestfile = 'x/test_doctest.txt'.replace('/', os.path.sep) + relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) + session.items = [ + StubPytestItem( + stub, + nodeid=doctestfile + '::test_doctest.txt', + name='test_doctest.txt', + location=(doctestfile, 0, '[doctest] test_doctest.txt'), + fspath=os.path.join(testroot, doctestfile), + kind='DoctestItem', + ), + # With --doctest-modules + StubPytestItem( + stub, + nodeid=relfile + '::test_eggs', + name='test_eggs', + location=(relfile, 0, '[doctest] test_eggs'), + fspath=os.path.join(testroot, relfile), + kind='DoctestItem', + ), + StubPytestItem( + stub, + nodeid=relfile + '::test_eggs.TestSpam', + name='test_eggs.TestSpam', + location=(relfile, 12, '[doctest] test_eggs.TestSpam'), + fspath=os.path.join(testroot, relfile), + kind='DoctestItem', + ), + StubPytestItem( + stub, + nodeid=relfile + '::test_eggs.TestSpam.TestEggs', + name='test_eggs.TestSpam.TestEggs', + location=(relfile, 27, '[doctest] test_eggs.TestSpam.TestEggs'), + fspath=os.path.join(testroot, relfile), + kind='DoctestItem', + ), + ] + collector = TestCollector(tests=discovered) + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id=doctestfile + '::test_doctest.txt', + name='test_doctest.txt', + path=TestPath( + root=testroot, + relfile=doctestfile, + func=None, + ), + source='{}:{}'.format(doctestfile, 0), + markers=[], + parentid=doctestfile, + ), + )), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id=relfile + '::test_eggs', + name='test_eggs', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 0), + markers=[], + parentid=relfile, + ), + )), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id=relfile + '::test_eggs.TestSpam', + name='test_eggs.TestSpam', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 12), + markers=[], + parentid=relfile, + ), + )), + ('discovered.add_test', None, dict( + suiteids=[], + test=TestInfo( + id=relfile + '::test_eggs.TestSpam.TestEggs', + name='test_eggs.TestSpam.TestEggs', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 27), + markers=[], + parentid=relfile, + ), + )), + ]) + + def test_nested_brackets(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) + session.items = [ + StubPytestItem( + stub, + nodeid=relfile + '::SpamTests::test_spam[a-[b]-c]', + name='test_spam[a-[b]-c]', + location=(relfile, 12, 'SpamTests.test_spam[a-[b]-c]'), + fspath=os.path.join(testroot, relfile), + function=FakeFunc('test_spam'), + ), + ] + collector = TestCollector(tests=discovered) + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[relfile + '::SpamTests'], + test=TestInfo( + id=relfile + '::SpamTests::test_spam[a-[b]-c]', + name='test_spam[a-[b]-c]', + path=TestPath( + root=testroot, + relfile=relfile, + func='SpamTests.test_spam', + sub=['[a-[b]-c]'], + ), + source='{}:{}'.format(relfile, 12), + markers=None, + parentid=relfile + '::SpamTests::test_spam', + ), + )), + ]) + + def test_nested_suite(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) + session.items = [ + StubPytestItem( + stub, + nodeid=relfile + '::SpamTests::Ham::Eggs::test_spam', + name='test_spam', + location=(relfile, 12, 'SpamTests.Ham.Eggs.test_spam'), + fspath=os.path.join(testroot, relfile), + function=FakeFunc('test_spam'), + ), + ] + collector = TestCollector(tests=discovered) + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[ + relfile + '::SpamTests', + relfile + '::SpamTests::Ham', + relfile + '::SpamTests::Ham::Eggs', + ], + test=TestInfo( + id=relfile + '::SpamTests::Ham::Eggs::test_spam', + name='test_spam', + path=TestPath( + root=testroot, + relfile=relfile, + func='SpamTests.Ham.Eggs.test_spam', + sub=None, + ), + source='{}:{}'.format(relfile, 12), + markers=None, + parentid=relfile + '::SpamTests::Ham::Eggs', + ), + )), + ]) + + def test_windows(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = r'c:\a\b\c' + relfile = r'X\Y\Z\test_eggs.py' + session.items = [ + StubPytestItem( + stub, + nodeid=relfile + '::SpamTests::test_spam', + name='test_spam', + location=('x/y/z/test_eggs.py', 12, 'SpamTests.test_spam'), + fspath=testroot + '\\' + relfile, + function=FakeFunc('test_spam'), + ), + ] + collector = TestCollector(tests=discovered) + if os.name != 'nt': + def normcase(path): + path = path.lower() + return path.replace('/', '\\') + collector.NORMCASE = normcase + collector.PATHSEP = '\\' + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[relfile + '::SpamTests'], + test=TestInfo( + id=relfile + '::SpamTests::test_spam', + name='test_spam', + path=TestPath( + root=testroot, + relfile=relfile, + func='SpamTests.test_spam', + sub=None, + ), + source='{}:{}'.format(relfile, 12), + markers=None, + parentid=relfile + '::SpamTests', + ), + )), + ]) + + def test_imported_test(self): + # pytest will even discover tests that were imported from + # another module! + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) + srcfile = 'x/y/z/_extern.py'.replace('/', os.path.sep) + session.items = [ + StubPytestItem( + stub, + nodeid=relfile + '::SpamTests::test_spam', + name='test_spam', + location=(srcfile, 12, 'SpamTests.test_spam'), + fspath=os.path.join(testroot, relfile), + function=FakeFunc('test_spam'), + ), + ] + collector = TestCollector(tests=discovered) + + collector.pytest_collection_finish(session) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('discovered.reset', None, None), + ('discovered.add_test', None, dict( + suiteids=[relfile + '::SpamTests'], + test=TestInfo( + id=relfile + '::SpamTests::test_spam', + name='test_spam', + path=TestPath( + root=testroot, + relfile=relfile, + func='SpamTests.test_spam', + sub=None, + ), + source='{}:{}'.format(srcfile, 12), + markers=None, + parentid=relfile + '::SpamTests', + ), + )), + ]) + + +class DiscoveredTestsTests(unittest.TestCase): + + def test_list(self): + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'test_spam.py' + relfileid = os.path.join('.', relfile) + tests = [ TestInfo( - id='test_spam.py::SpamTests::test_one', - name='test_one', + id=relfile + '::test_each[10-10]', + name='test_each[10-10]', path=TestPath( root=testroot, - relfile=relfile1, - func='SpamTests.test_one', - sub=None, + relfile=relfile, + func='test_each', + sub=['[10-10]'], ), - lineno=12, + source='{}:{}'.format(relfile, 10), markers=None, + parentid=relfile + '::test_each', ), TestInfo( - id='test_spam.py::SpamTests::test_other', - name='test_other', + id=relfile + '::All::BasicTests::test_first', + name='test_first', path=TestPath( root=testroot, - relfile=relfile1, - func='SpamTests.test_other', + relfile=relfile, + func='All.BasicTests.test_first', sub=None, ), - lineno=19, + source='{}:{}'.format(relfile, 61), markers=None, + parentid=relfile + '::All::BasicTests', + ), + ] + allsuiteids = [ + [], + [relfile + '::All', + relfile + '::All::BasicTests', + ], + ] + expected = [test._replace(id=os.path.join('.', test.id), + parentid=os.path.join('.', test.parentid)) + for test in tests] + discovered = DiscoveredTests() + for test, suiteids in zip(tests, allsuiteids): + discovered.add_test(test, suiteids) + size = len(discovered) + items = [discovered[0], discovered[1]] + snapshot = list(discovered) + + self.maxDiff = None + self.assertEqual(size, 2) + self.assertEqual(items, expected) + self.assertEqual(snapshot, expected) + + def test_reset(self): + testroot = '/a/b/c'.replace('/', os.path.sep) + discovered = DiscoveredTests() + discovered.add_test( + TestInfo( + id='test_spam.py::test_each', + name='test_each', + path=TestPath( + root=testroot, + relfile='test_spam.py', + func='test_each', + ), + source='{}:{}'.format('test_spam.py', 10), + markers=[], + parentid='test_spam.py', ), + []) + + before = len(discovered), len(discovered.parents) + discovered.reset() + after = len(discovered), len(discovered.parents) + + self.assertEqual(before, (1, 2)) + self.assertEqual(after, (0, 0)) + + def test_parents(self): + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'x/y/z/test_spam.py'.replace('/', os.path.sep) + relfileid = os.path.join('.', relfile) + tests = [ TestInfo( - id='test_spam.py::test_all', - name='test_all', + id=relfile + '::test_each[10-10]', + name='test_each[10-10]', path=TestPath( root=testroot, - relfile=relfile1, - func='test_all', - sub=None, + relfile=relfile, + func='test_each', + sub=['[10-10]'], ), - lineno=144, + source='{}:{}'.format(relfile, 10), markers=None, + parentid=relfile + '::test_each', ), TestInfo( - id=relfile2 + '::All::BasicTests::test_first', + id=relfile + '::All::BasicTests::test_first', name='test_first', path=TestPath( root=testroot, - relfile=relfile2, + relfile=relfile, func='All.BasicTests.test_first', sub=None, ), - lineno=31, + source='{}:{}'.format(relfile, 61), markers=None, + parentid=relfile + '::All::BasicTests', + ), + ] + allsuiteids = [ + [], + [relfile + '::All', + relfile + '::All::BasicTests', + ], + ] + discovered = DiscoveredTests() + for test, suiteids in zip(tests, allsuiteids): + discovered.add_test(test, suiteids) + + parents = discovered.parents + + self.maxDiff = None + self.assertEqual(parents, [ + ParentInfo( + id='.', + kind='folder', + name=testroot, + ), + ParentInfo( + id='./x'.replace('/', os.path.sep), + kind='folder', + name='x', + root=testroot, + parentid='.', + ), + ParentInfo( + id='./x/y'.replace('/', os.path.sep), + kind='folder', + name='y', + root=testroot, + parentid='./x'.replace('/', os.path.sep), + ), + ParentInfo( + id='./x/y/z'.replace('/', os.path.sep), + kind='folder', + name='z', + root=testroot, + parentid='./x/y'.replace('/', os.path.sep), + ), + ParentInfo( + id=relfileid, + kind='file', + name=os.path.basename(relfile), + root=testroot, + parentid=os.path.dirname(relfileid), + ), + ParentInfo( + id=relfileid + '::All', + kind='suite', + name='All', + root=testroot, + parentid=relfileid, + ), + ParentInfo( + id=relfileid + '::All::BasicTests', + kind='suite', + name='BasicTests', + root=testroot, + parentid=relfileid + '::All', + ), + ParentInfo( + id=relfileid + '::test_each', + kind='function', + name='test_each', + root=testroot, + parentid=relfileid, + ), + ]) + + def test_add_test_simple(self): + testroot = '/a/b/c'.replace('/', os.path.sep) + test = TestInfo( + id='test_spam.py::test_spam', + name='test_spam', + path=TestPath( + root=testroot, + relfile='test_spam.py', + func='test_spam', + ), + source='{}:{}'.format('test_spam.py', 11), + markers=[], + parentid='test_spam.py', + ) + expected = test._replace(id=os.path.join('.', test.id), + parentid=os.path.join('.', test.parentid)) + discovered = DiscoveredTests() + + before = list(discovered), discovered.parents + discovered.add_test(test, []) + after = list(discovered), discovered.parents + + self.maxDiff = None + self.assertEqual(before, ([], [])) + self.assertEqual(after, ([expected], [ + ParentInfo( + id='.', + kind='folder', + name=testroot, ), + ParentInfo( + id=os.path.join('.', 'test_spam.py'), + kind='file', + name='test_spam.py', + root=testroot, + parentid='.', + ), + ])) + + def test_multiroot(self): + # the first root + testroot1 = '/a/b/c'.replace('/', os.path.sep) + relfile1 = 'test_spam.py' + relfileid1 = os.path.join('.', relfile1) + alltests = [ TestInfo( - id=relfile2 + '::All::BasicTests::test_each[1+2-3]', - name='test_each[1+2-3]', + id=relfile1 + '::test_spam', + name='test_spam', path=TestPath( - root=testroot, + root=testroot1, + relfile=relfile1, + func='test_spam', + ), + source='{}:{}'.format(relfile1, 10), + markers=[], + parentid=relfile1, + ), + ] + allsuiteids = [ + [], + ] + # the second root + testroot2 = '/x/y/z'.replace('/', os.path.sep) + relfile2 = 'w/test_eggs.py' + relfileid2 = os.path.join('.', relfile2) + alltests.extend([ + TestInfo( + id=relfile2 + 'BasicTests::test_first', + name='test_first', + path=TestPath( + root=testroot2, relfile=relfile2, - func='All.BasicTests.test_each', - sub=['[1+2-3]'], + func='BasicTests.test_first', ), - lineno=62, - markers=['expected-failure', 'skip', 'skip-if'], + source='{}:{}'.format(relfile2, 61), + markers=[], + parentid=relfile2 + '::BasicTests', ), ]) - self.assertEqual(stub.calls, []) + allsuiteids.extend([ + [relfile2 + '::BasicTests', + ], + ]) - def test_finish(self): + discovered = DiscoveredTests() + for test, suiteids in zip(alltests, allsuiteids): + discovered.add_test(test, suiteids) + tests = list(discovered) + parents = discovered.parents + + self.maxDiff = None + self.assertEqual(tests, [ + # the first root + TestInfo( + id=relfileid1 + '::test_spam', + name='test_spam', + path=TestPath( + root=testroot1, + relfile=relfile1, + func='test_spam', + ), + source='{}:{}'.format(relfile1, 10), + markers=[], + parentid=relfileid1, + ), + # the secondroot + TestInfo( + id=relfileid2 + 'BasicTests::test_first', + name='test_first', + path=TestPath( + root=testroot2, + relfile=relfile2, + func='BasicTests.test_first', + ), + source='{}:{}'.format(relfile2, 61), + markers=[], + parentid=relfileid2 + '::BasicTests', + ), + ]) + self.assertEqual(parents, [ + # the first root + ParentInfo( + id='.', + kind='folder', + name=testroot1, + ), + ParentInfo( + id=relfileid1, + kind='file', + name=os.path.basename(relfile1), + root=testroot1, + parentid=os.path.dirname(relfileid1), + ), + # the secondroot + ParentInfo( + id='.', + kind='folder', + name=testroot2, + ), + ParentInfo( + id='./w'.replace('/', os.path.sep), + kind='folder', + name='w', + root=testroot2, + parentid='.', + ), + ParentInfo( + id=relfileid2, + kind='file', + name=os.path.basename(relfile2), + root=testroot2, + parentid=os.path.dirname(relfileid2), + ), + ParentInfo( + id=relfileid2 + '::BasicTests', + kind='suite', + name='BasicTests', + root=testroot2, + parentid=relfileid2, + ), + ]) + + def test_doctest(self): stub = Stub() - session = StubPytestSession(stub) testroot = '/a/b/c'.replace('/', os.path.sep) - relfile = 'x/y/z/test_eggs.py'.replace('/', os.path.sep) - session.items = [ - StubPytestItem( - stub, - nodeid=relfile + '::SpamTests::test_spam', - name='test_spam', - location=(relfile, 12, 'SpamTests.test_spam'), - fspath=os.path.join(testroot, relfile), - function=FakeFunc('test_spam'), + doctestfile = './x/test_doctest.txt'.replace('/', os.path.sep) + relfile = './x/y/z/test_eggs.py'.replace('/', os.path.sep) + alltests = [ + TestInfo( + id=doctestfile + '::test_doctest.txt', + name='test_doctest.txt', + path=TestPath( + root=testroot, + relfile=doctestfile, + func=None, + ), + source='{}:{}'.format(doctestfile, 0), + markers=[], + parentid=doctestfile, + ), + # With --doctest-modules + TestInfo( + id=relfile + '::test_eggs', + name='test_eggs', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 0), + markers=[], + parentid=relfile, + ), + TestInfo( + id=relfile + '::test_eggs.TestSpam', + name='test_eggs.TestSpam', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 12), + markers=[], + parentid=relfile, + ), + TestInfo( + id=relfile + '::test_eggs.TestSpam.TestEggs', + name='test_eggs.TestSpam.TestEggs', + path=TestPath( + root=testroot, + relfile=relfile, + func=None, + ), + source='{}:{}'.format(relfile, 27), + markers=[], + parentid=relfile, ), ] - collector = TestCollector() + discovered = DiscoveredTests() - collector.pytest_collection_finish(session) + for test in alltests: + discovered.add_test(test, []) + tests = list(discovered) + parents = discovered.parents self.maxDiff = None - self.assertEqual(collector.discovered, [ + self.assertEqual(tests, alltests) + self.assertEqual(parents, [ + ParentInfo( + id='.', + kind='folder', + name=testroot, + ), + ParentInfo( + id='./x'.replace('/', os.path.sep), + kind='folder', + name='x', + root=testroot, + parentid='.', + ), + ParentInfo( + id=doctestfile, + kind='file', + name=os.path.basename(doctestfile), + root=testroot, + parentid=os.path.dirname(doctestfile), + ), + ParentInfo( + id='./x/y'.replace('/', os.path.sep), + kind='folder', + name='y', + root=testroot, + parentid='./x'.replace('/', os.path.sep), + ), + ParentInfo( + id='./x/y/z'.replace('/', os.path.sep), + kind='folder', + name='z', + root=testroot, + parentid='./x/y'.replace('/', os.path.sep), + ), + ParentInfo( + id=relfile, + kind='file', + name=os.path.basename(relfile), + root=testroot, + parentid=os.path.dirname(relfile), + ), + ]) + + def test_nested_suite_simple(self): + stub = Stub() + discovered = StubDiscoveredTests(stub) + session = StubPytestSession(stub) + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = './test_eggs.py'.replace('/', os.path.sep) + alltests = [ TestInfo( - id=relfile + '::SpamTests::test_spam', + id=relfile + '::TestOuter::TestInner::test_spam', name='test_spam', path=TestPath( root=testroot, relfile=relfile, - func='SpamTests.test_spam', - sub=None, + func='TestOuter.TestInner.test_spam', + ), + source='{}:{}'.format(relfile, 10), + markers=None, + parentid=relfile + '::TestOuter::TestInner', + ), + TestInfo( + id=relfile + '::TestOuter::TestInner::test_eggs', + name='test_eggs', + path=TestPath( + root=testroot, + relfile=relfile, + func='TestOuter.TestInner.test_eggs', ), - lineno=12, + source='{}:{}'.format(relfile, 21), markers=None, + parentid=relfile + '::TestOuter::TestInner', + ), + ] + allsuiteids = [ + [relfile + '::TestOuter', + relfile + '::TestOuter::TestInner', + ], + [relfile + '::TestOuter', + relfile + '::TestOuter::TestInner', + ], + ] + + discovered = DiscoveredTests() + for test, suiteids in zip(alltests, allsuiteids): + discovered.add_test(test, suiteids) + tests = list(discovered) + parents = discovered.parents + + self.maxDiff = None + self.assertEqual(tests, alltests) + self.assertEqual(parents, [ + ParentInfo( + id='.', + kind='folder', + name=testroot, + ), + ParentInfo( + id=relfile, + kind='file', + name=os.path.basename(relfile), + root=testroot, + parentid=os.path.dirname(relfile), + ), + ParentInfo( + id=relfile + '::TestOuter', + kind='suite', + name='TestOuter', + root=testroot, + parentid=relfile, + ), + ParentInfo( + id=relfile + '::TestOuter::TestInner', + kind='suite', + name='TestInner', + root=testroot, + parentid=relfile + '::TestOuter', ), ]) - self.assertEqual(stub.calls, []) diff --git a/pythonFiles/tests/testing_tools/adapter/test_report.py b/pythonFiles/tests/testing_tools/adapter/test_report.py index 2a844b50d4e8..4628c0719729 100644 --- a/pythonFiles/tests/testing_tools/adapter/test_report.py +++ b/pythonFiles/tests/testing_tools/adapter/test_report.py @@ -6,22 +6,736 @@ import unittest from ...util import StubProxy -from testing_tools.adapter.info import TestInfo, TestPath +from testing_tools.adapter.info import TestInfo, TestPath, ParentInfo from testing_tools.adapter.report import report_discovered class StubSender(StubProxy): def send(self, outstr): - self.add_call('send', (outstr,), None) + self.add_call('send', (json.loads(outstr),), None) ################################## # tests -class ReportTests(unittest.TestCase): +class ReportDiscoveredTests(unittest.TestCase): def test_basic(self): + stub = StubSender() + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile = 'test_spam.py' + tests = [ + TestInfo( + id='test#1', + name='test_spam', + path=TestPath( + root=testroot, + relfile=relfile, + func='test_spam', + ), + source='{}:{}'.format(relfile, 10), + markers=[], + parentid='file#1', + ), + ] + parents = [ + ParentInfo( + id='', + kind='folder', + name=testroot, + ), + ParentInfo( + id='file#1', + kind='file', + name=relfile, + root=testroot, + parentid='', + ), + ] + expected = [{ + 'rootid': '', + 'root': testroot, + 'parents': [ + {'id': 'file#1', + 'kind': 'file', + 'name': relfile, + 'parentid': '', + }, + ], + 'tests': [{ + 'id': 'test#1', + 'name': 'test_spam', + 'source': '{}:{}'.format(relfile, 10), + 'markers': [], + 'parentid': 'file#1', + }], + }] + + report_discovered(tests, parents, _send=stub.send) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('send', (expected,), None), + ]) + + def test_multiroot(self): + stub = StubSender() + # the first root + testroot1 = '/a/b/c'.replace('/', os.path.sep) + relfile1 = 'test_spam.py' + relfileid1 = os.path.join('.', relfile1) + tests = [ + TestInfo( + id=relfileid1 + '::test_spam', + name='test_spam', + path=TestPath( + root=testroot1, + relfile=relfile1, + func='test_spam', + ), + source='{}:{}'.format(relfile1, 10), + markers=[], + parentid=relfileid1, + ), + ] + parents = [ + ParentInfo( + id='.', + kind='folder', + name=testroot1, + ), + ParentInfo( + id=relfileid1, + kind='file', + name=os.path.basename(relfile1), + root=testroot1, + parentid=os.path.dirname(relfileid1), + ), + ] + expected = [ + {'rootid': '.', + 'root': testroot1, + 'parents': [ + {'id': relfileid1, + 'kind': 'file', + 'name': relfile1, + 'parentid': '.', + }, + ], + 'tests': [{ + 'id': relfileid1 + '::test_spam', + 'name': 'test_spam', + 'source': '{}:{}'.format(relfile1, 10), + 'markers': [], + 'parentid': relfileid1, + }], + }, + ] + # the second root + testroot2 = '/x/y/z'.replace('/', os.path.sep) + relfile2 = 'w/test_eggs.py' + relfileid2 = os.path.join('.', relfile2) + tests.extend([ + TestInfo( + id=relfileid2 + '::BasicTests::test_first', + name='test_first', + path=TestPath( + root=testroot2, + relfile=relfile2, + func='BasicTests.test_first', + ), + source='{}:{}'.format(relfile2, 61), + markers=[], + parentid=relfileid2 + '::BasicTests', + ), + ]) + parents.extend([ + ParentInfo( + id='.', + kind='folder', + name=testroot2, + ), + ParentInfo( + id='./w'.replace('/', os.path.sep), + kind='folder', + name='w', + root=testroot2, + parentid='.', + ), + ParentInfo( + id=relfileid2, + kind='file', + name=os.path.basename(relfile2), + root=testroot2, + parentid=os.path.dirname(relfileid2), + ), + ParentInfo( + id=relfileid2 + '::BasicTests', + kind='suite', + name='BasicTests', + root=testroot2, + parentid=relfileid2, + ), + ]) + expected.extend([ + {'rootid': '.', + 'root': testroot2, + 'parents': [ + {'id': os.path.dirname(relfileid2), + 'kind': 'folder', + 'name': 'w', + 'parentid': '.', + }, + {'id': relfileid2, + 'kind': 'file', + 'name': os.path.basename(relfile2), + 'parentid': os.path.dirname(relfileid2), + }, + {'id': relfileid2 + '::BasicTests', + 'kind': 'suite', + 'name': 'BasicTests', + 'parentid': relfileid2, + }, + ], + 'tests': [{ + 'id': relfileid2 + '::BasicTests::test_first', + 'name': 'test_first', + 'source': '{}:{}'.format(relfile2, 61), + 'markers': [], + 'parentid': relfileid2 + '::BasicTests', + }], + }, + ]) + + report_discovered(tests, parents, _send=stub.send) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('send', (expected,), None), + ]) + + def test_complex(self): + """ + /a/b/c/ + test_ham.py + MySuite + test_x1 + test_x2 + /a/b/e/f/g/ + w/ + test_ham.py + test_ham1 + HamTests + test_uh_oh + test_whoa + MoreHam + test_yay + sub1 + sub2 + sub3 + test_eggs.py + SpamTests + test_okay + x/ + y/ + a/ + test_spam.py + SpamTests + test_okay + b/ + test_spam.py + SpamTests + test_okay + test_spam.py + SpamTests + test_okay + """ + stub = StubSender() + testroot = '/a/b/c'.replace('/', os.path.sep) + relfile1 = './test_ham.py'.replace('/', os.path.sep) + relfile2 = './test_spam.py'.replace('/', os.path.sep) + relfile3 = './w/test_ham.py'.replace('/', os.path.sep) + relfile4 = './w/test_eggs.py'.replace('/', os.path.sep) + relfile5 = './x/y/a/test_spam.py'.replace('/', os.path.sep) + relfile6 = './x/y/b/test_spam.py'.replace('/', os.path.sep) + tests = [ + TestInfo( + id=relfile1 + '::MySuite::test_x1', + name='test_x1', + path=TestPath( + root=testroot, + relfile=relfile1, + func='MySuite.test_x1', + ), + source='{}:{}'.format(relfile1, 10), + markers=None, + parentid=relfile1 + '::MySuite', + ), + TestInfo( + id=relfile1 + '::MySuite::test_x2', + name='test_x2', + path=TestPath( + root=testroot, + relfile=relfile1, + func='MySuite.test_x2', + ), + source='{}:{}'.format(relfile1, 21), + markers=None, + parentid=relfile1 + '::MySuite', + ), + TestInfo( + id=relfile2 + '::SpamTests::test_okay', + name='test_okay', + path=TestPath( + root=testroot, + relfile=relfile2, + func='SpamTests.test_okay', + ), + source='{}:{}'.format(relfile2, 17), + markers=None, + parentid=relfile2 + '::SpamTests', + ), + TestInfo( + id=relfile3 + '::test_ham1', + name='test_ham1', + path=TestPath( + root=testroot, + relfile=relfile3, + func='test_ham1', + ), + source='{}:{}'.format(relfile3, 8), + markers=None, + parentid=relfile3, + ), + TestInfo( + id=relfile3 + '::HamTests::test_uh_oh', + name='test_uh_oh', + path=TestPath( + root=testroot, + relfile=relfile3, + func='HamTests.test_uh_oh', + ), + source='{}:{}'.format(relfile3, 19), + markers=['expected-failure'], + parentid=relfile3 + '::HamTests', + ), + TestInfo( + id=relfile3 + '::HamTests::test_whoa', + name='test_whoa', + path=TestPath( + root=testroot, + relfile=relfile3, + func='HamTests.test_whoa', + ), + source='{}:{}'.format(relfile3, 35), + markers=None, + parentid=relfile3 + '::HamTests', + ), + TestInfo( + id=relfile3 + '::MoreHam::test_yay[1-2]', + name='test_yay[1-2]', + path=TestPath( + root=testroot, + relfile=relfile3, + func='MoreHam.test_yay', + sub=['[1-2]'], + ), + source='{}:{}'.format(relfile3, 57), + markers=None, + parentid=relfile3 + '::MoreHam::test_yay', + ), + TestInfo( + id=relfile3 + '::MoreHam::test_yay[1-2][3-4]', + name='test_yay[1-2][3-4]', + path=TestPath( + root=testroot, + relfile=relfile3, + func='MoreHam.test_yay', + sub=['[1-2]', '[3=4]'], + ), + source='{}:{}'.format(relfile3, 72), + markers=None, + parentid=relfile3 + '::MoreHam::test_yay[1-2]', + ), + TestInfo( + id=relfile4 + '::SpamTests::test_okay', + name='test_okay', + path=TestPath( + root=testroot, + relfile=relfile4, + func='SpamTests.test_okay', + ), + source='{}:{}'.format(relfile4, 15), + markers=None, + parentid=relfile4 + '::SpamTests', + ), + TestInfo( + id=relfile5 + '::SpamTests::test_okay', + name='test_okay', + path=TestPath( + root=testroot, + relfile=relfile5, + func='SpamTests.test_okay', + ), + source='{}:{}'.format(relfile5, 12), + markers=None, + parentid=relfile5 + '::SpamTests', + ), + TestInfo( + id=relfile6 + '::SpamTests::test_okay', + name='test_okay', + path=TestPath( + root=testroot, + relfile=relfile6, + func='SpamTests.test_okay', + ), + source='{}:{}'.format(relfile6, 27), + markers=None, + parentid=relfile6 + '::SpamTests', + ), + ] + parents = [ + ParentInfo( + id='.', + kind='folder', + name=testroot, + ), + + ParentInfo( + id=relfile1, + kind='file', + name=os.path.basename(relfile1), + root=testroot, + parentid='.', + ), + ParentInfo( + id=relfile1 + '::MySuite', + kind='suite', + name='MySuite', + root=testroot, + parentid=relfile1, + ), + + ParentInfo( + id=relfile2, + kind='file', + name=os.path.basename(relfile2), + root=testroot, + parentid='.', + ), + ParentInfo( + id=relfile2 + '::SpamTests', + kind='suite', + name='SpamTests', + root=testroot, + parentid=relfile2, + ), + + ParentInfo( + id='./w'.replace('/', os.path.sep), + kind='folder', + name='w', + root=testroot, + parentid='.', + ), + ParentInfo( + id=relfile3, + kind='file', + name=os.path.basename(relfile3), + root=testroot, + parentid=os.path.dirname(relfile3), + ), + ParentInfo( + id=relfile3 + '::HamTests', + kind='suite', + name='HamTests', + root=testroot, + parentid=relfile3, + ), + ParentInfo( + id=relfile3 + '::MoreHam', + kind='suite', + name='MoreHam', + root=testroot, + parentid=relfile3, + ), + ParentInfo( + id=relfile3 + '::MoreHam::test_yay', + kind='function', + name='test_yay', + root=testroot, + parentid=relfile3 + '::MoreHam', + ), + ParentInfo( + id=relfile3 + '::MoreHam::test_yay[1-2]', + kind='subtest', + name='test_yay[1-2]', + root=testroot, + parentid=relfile3 + '::MoreHam::test_yay', + ), + + ParentInfo( + id=relfile4, + kind='file', + name=os.path.basename(relfile4), + root=testroot, + parentid=os.path.dirname(relfile4), + ), + ParentInfo( + id=relfile4 + '::SpamTests', + kind='suite', + name='SpamTests', + root=testroot, + parentid=relfile4, + ), + + ParentInfo( + id='./x'.replace('/', os.path.sep), + kind='folder', + name='x', + root=testroot, + parentid='.', + ), + ParentInfo( + id='./x/y'.replace('/', os.path.sep), + kind='folder', + name='y', + root=testroot, + parentid='./x'.replace('/', os.path.sep), + ), + ParentInfo( + id='./x/y/a'.replace('/', os.path.sep), + kind='folder', + name='a', + root=testroot, + parentid='./x/y'.replace('/', os.path.sep), + ), + ParentInfo( + id=relfile5, + kind='file', + name=os.path.basename(relfile5), + root=testroot, + parentid=os.path.dirname(relfile5), + ), + ParentInfo( + id=relfile5 + '::SpamTests', + kind='suite', + name='SpamTests', + root=testroot, + parentid=relfile5, + ), + + ParentInfo( + id='./x/y/b'.replace('/', os.path.sep), + kind='folder', + name='b', + root=testroot, + parentid='./x/y'.replace('/', os.path.sep), + ), + ParentInfo( + id=relfile6, + kind='file', + name=os.path.basename(relfile6), + root=testroot, + parentid=os.path.dirname(relfile6), + ), + ParentInfo( + id=relfile6 + '::SpamTests', + kind='suite', + name='SpamTests', + root=testroot, + parentid=relfile6, + ), + ] + expected = [{ + 'rootid': '.', + 'root': testroot, + 'parents': [ + {'id': relfile1, + 'kind': 'file', + 'name': os.path.basename(relfile1), + 'parentid': '.', + }, + {'id': relfile1 + '::MySuite', + 'kind': 'suite', + 'name': 'MySuite', + 'parentid': relfile1, + }, + + {'id': relfile2, + 'kind': 'file', + 'name': os.path.basename(relfile2), + 'parentid': '.', + }, + {'id': relfile2 + '::SpamTests', + 'kind': 'suite', + 'name': 'SpamTests', + 'parentid': relfile2, + }, + + {'id': './w'.replace('/', os.path.sep), + 'kind': 'folder', + 'name': 'w', + 'parentid': '.', + }, + {'id': relfile3, + 'kind': 'file', + 'name': os.path.basename(relfile3), + 'parentid': os.path.dirname(relfile3), + }, + {'id': relfile3 + '::HamTests', + 'kind': 'suite', + 'name': 'HamTests', + 'parentid': relfile3, + }, + {'id': relfile3 + '::MoreHam', + 'kind': 'suite', + 'name': 'MoreHam', + 'parentid': relfile3, + }, + {'id': relfile3 + '::MoreHam::test_yay', + 'kind': 'function', + 'name': 'test_yay', + 'parentid': relfile3 + '::MoreHam', + }, + {'id': relfile3 + '::MoreHam::test_yay[1-2]', + 'kind': 'subtest', + 'name': 'test_yay[1-2]', + 'parentid': relfile3 + '::MoreHam::test_yay', + }, + + {'id': relfile4, + 'kind': 'file', + 'name': os.path.basename(relfile4), + 'parentid': os.path.dirname(relfile4), + }, + {'id': relfile4 + '::SpamTests', + 'kind': 'suite', + 'name': 'SpamTests', + 'parentid': relfile4, + }, + + {'id': './x'.replace('/', os.path.sep), + 'kind': 'folder', + 'name': 'x', + 'parentid': '.', + }, + {'id': './x/y'.replace('/', os.path.sep), + 'kind': 'folder', + 'name': 'y', + 'parentid': './x'.replace('/', os.path.sep), + }, + {'id': './x/y/a'.replace('/', os.path.sep), + 'kind': 'folder', + 'name': 'a', + 'parentid': './x/y'.replace('/', os.path.sep), + }, + {'id': relfile5, + 'kind': 'file', + 'name': os.path.basename(relfile5), + 'parentid': os.path.dirname(relfile5), + }, + {'id': relfile5 + '::SpamTests', + 'kind': 'suite', + 'name': 'SpamTests', + 'parentid': relfile5, + }, + + {'id': './x/y/b'.replace('/', os.path.sep), + 'kind': 'folder', + 'name': 'b', + 'parentid': './x/y'.replace('/', os.path.sep), + }, + {'id': relfile6, + 'kind': 'file', + 'name': os.path.basename(relfile6), + 'parentid': os.path.dirname(relfile6), + }, + {'id': relfile6 + '::SpamTests', + 'kind': 'suite', + 'name': 'SpamTests', + 'parentid': relfile6, + }, + ], + 'tests': [ + {'id': relfile1 + '::MySuite::test_x1', + 'name': 'test_x1', + 'source': '{}:{}'.format(relfile1, 10), + 'markers': [], + 'parentid': relfile1 + '::MySuite', + }, + {'id': relfile1 + '::MySuite::test_x2', + 'name': 'test_x2', + 'source': '{}:{}'.format(relfile1, 21), + 'markers': [], + 'parentid': relfile1 + '::MySuite', + }, + {'id': relfile2 + '::SpamTests::test_okay', + 'name': 'test_okay', + 'source': '{}:{}'.format(relfile2, 17), + 'markers': [], + 'parentid': relfile2 + '::SpamTests', + }, + {'id': relfile3 + '::test_ham1', + 'name': 'test_ham1', + 'source': '{}:{}'.format(relfile3, 8), + 'markers': [], + 'parentid': relfile3, + }, + {'id': relfile3 + '::HamTests::test_uh_oh', + 'name': 'test_uh_oh', + 'source': '{}:{}'.format(relfile3, 19), + 'markers': ['expected-failure'], + 'parentid': relfile3 + '::HamTests', + }, + {'id': relfile3 + '::HamTests::test_whoa', + 'name': 'test_whoa', + 'source': '{}:{}'.format(relfile3, 35), + 'markers': [], + 'parentid': relfile3 + '::HamTests', + }, + {'id': relfile3 + '::MoreHam::test_yay[1-2]', + 'name': 'test_yay[1-2]', + 'source': '{}:{}'.format(relfile3, 57), + 'markers': [], + 'parentid': relfile3 + '::MoreHam::test_yay', + }, + {'id': relfile3 + '::MoreHam::test_yay[1-2][3-4]', + 'name': 'test_yay[1-2][3-4]', + 'source': '{}:{}'.format(relfile3, 72), + 'markers': [], + 'parentid': relfile3 + '::MoreHam::test_yay[1-2]', + }, + {'id': relfile4 + '::SpamTests::test_okay', + 'name': 'test_okay', + 'source': '{}:{}'.format(relfile4, 15), + 'markers': [], + 'parentid': relfile4 + '::SpamTests', + }, + {'id': relfile5 + '::SpamTests::test_okay', + 'name': 'test_okay', + 'source': '{}:{}'.format(relfile5, 12), + 'markers': [], + 'parentid': relfile5 + '::SpamTests', + }, + {'id': relfile6 + '::SpamTests::test_okay', + 'name': 'test_okay', + 'source': '{}:{}'.format(relfile6, 27), + 'markers': [], + 'parentid': relfile6 + '::SpamTests', + }, + ], + }] + + report_discovered(tests, parents, _send=stub.send) + + self.maxDiff = None + self.assertEqual(stub.calls, [ + ('send', (expected,), None), + ]) + + def test_simple_basic(self): stub = StubSender() testroot = '/a/b/c'.replace('/', os.path.sep) relfile = 'x/y/z/test_spam.py'.replace('/', os.path.sep) @@ -35,10 +749,12 @@ def test_basic(self): func='MySuite.test_spam_1', sub=None, ), - lineno=10, + source='{}:{}'.format(relfile, 10), markers=None, + parentid='suite#1', ), ] + parents = None expected = [{ 'id': 'test#1', 'name': 'test_spam_1', @@ -47,16 +763,18 @@ def test_basic(self): 'lineno': 10, 'testfunc': 'MySuite.test_spam_1', 'subtest': None, - 'markers': None, + 'markers': [], }] - report_discovered(tests, _send=stub.send) + report_discovered(tests, parents, simple=True, + _send=stub.send) + self.maxDiff = None self.assertEqual(stub.calls, [ - ('send', (json.dumps(expected),), None), + ('send', (expected,), None), ]) - def test_complex(self): + def test_simple_complex(self): """ /a/b/c/ test_ham.py @@ -112,8 +830,9 @@ def test_complex(self): func='MySuite.test_x1', sub=None, ), - lineno=10, + source='{}:{}'.format(relfile1, 10), markers=None, + parentid='suite#1', ), TestInfo( id='test#2', @@ -124,8 +843,9 @@ def test_complex(self): func='MySuite.test_x2', sub=None, ), - lineno=21, + source='{}:{}'.format(relfile1, 21), markers=None, + parentid='suite#1', ), # under second root folder TestInfo( @@ -137,8 +857,9 @@ def test_complex(self): func='SpamTests.test_okay', sub=None, ), - lineno=17, + source='{}:{}'.format(relfile2, 17), markers=None, + parentid='suite#2', ), TestInfo( id='test#4', @@ -149,8 +870,9 @@ def test_complex(self): func='test_ham1', sub=None, ), - lineno=8, + source='{}:{}'.format(relfile3, 8), markers=None, + parentid='file#3', ), TestInfo( id='test#5', @@ -161,8 +883,9 @@ def test_complex(self): func='HamTests.test_uh_oh', sub=None, ), - lineno=19, + source='{}:{}'.format(relfile3, 19), markers=['expected-failure'], + parentid='suite#3', ), TestInfo( id='test#6', @@ -173,8 +896,9 @@ def test_complex(self): func='HamTests.test_whoa', sub=None, ), - lineno=35, + source='{}:{}'.format(relfile3, 35), markers=None, + parentid='suite#3', ), TestInfo( id='test#7', @@ -185,8 +909,9 @@ def test_complex(self): func='MoreHam.test_yay', sub=['sub1'], ), - lineno=57, + source='{}:{}'.format(relfile3, 57), markers=None, + parentid='suite#4', ), TestInfo( id='test#8', @@ -197,8 +922,9 @@ def test_complex(self): func='MoreHam.test_yay', sub=['sub2', 'sub3'], ), - lineno=72, + source='{}:{}'.format(relfile3, 72), markers=None, + parentid='suite#3', ), TestInfo( id='test#9', @@ -209,8 +935,9 @@ def test_complex(self): func='SpamTests.test_okay', sub=None, ), - lineno=15, + source='{}:{}'.format(relfile4, 15), markers=None, + parentid='suite#5', ), TestInfo( id='test#10', @@ -221,8 +948,9 @@ def test_complex(self): func='SpamTests.test_okay', sub=None, ), - lineno=12, + source='{}:{}'.format(relfile5, 12), markers=None, + parentid='suite#6', ), TestInfo( id='test#11', @@ -233,8 +961,9 @@ def test_complex(self): func='SpamTests.test_okay', sub=None, ), - lineno=27, + source='{}:{}'.format(relfile6, 27), markers=None, + parentid='suite#7', ), ] expected = [{ @@ -245,7 +974,7 @@ def test_complex(self): 'lineno': 10, 'testfunc': 'MySuite.test_x1', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#2', 'name': 'test_x2', @@ -254,7 +983,7 @@ def test_complex(self): 'lineno': 21, 'testfunc': 'MySuite.test_x2', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#3', 'name': 'test_okay', @@ -263,7 +992,7 @@ def test_complex(self): 'lineno': 17, 'testfunc': 'SpamTests.test_okay', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#4', 'name': 'test_ham1', @@ -272,7 +1001,7 @@ def test_complex(self): 'lineno': 8, 'testfunc': 'test_ham1', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#5', 'name': 'test_uh_oh', @@ -290,7 +1019,7 @@ def test_complex(self): 'lineno': 35, 'testfunc': 'HamTests.test_whoa', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#7', 'name': 'test_yay (sub1)', @@ -299,7 +1028,7 @@ def test_complex(self): 'lineno': 57, 'testfunc': 'MoreHam.test_yay', 'subtest': ['sub1'], - 'markers': None, + 'markers': [], }, { 'id': 'test#8', 'name': 'test_yay (sub2) (sub3)', @@ -308,7 +1037,7 @@ def test_complex(self): 'lineno': 72, 'testfunc': 'MoreHam.test_yay', 'subtest': ['sub2', 'sub3'], - 'markers': None, + 'markers': [], }, { 'id': 'test#9', 'name': 'test_okay', @@ -317,7 +1046,7 @@ def test_complex(self): 'lineno': 15, 'testfunc': 'SpamTests.test_okay', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#10', 'name': 'test_okay', @@ -326,7 +1055,7 @@ def test_complex(self): 'lineno': 12, 'testfunc': 'SpamTests.test_okay', 'subtest': None, - 'markers': None, + 'markers': [], }, { 'id': 'test#11', 'name': 'test_okay', @@ -335,12 +1064,14 @@ def test_complex(self): 'lineno': 27, 'testfunc': 'SpamTests.test_okay', 'subtest': None, - 'markers': None, + 'markers': [], }] + parents = None - report_discovered(tests, _send=stub.send) + report_discovered(tests, parents, simple=True, + _send=stub.send) self.maxDiff = None self.assertEqual(stub.calls, [ - ('send', (json.dumps(expected),), None), + ('send', (expected,), None), ]) diff --git a/src/test/pythonFiles/testFiles/standard/tests/__init__.py b/src/test/pythonFiles/testFiles/standard/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1