From 797fff165e7c04d571fff2bbcda59fb94688e108 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Tue, 16 Sep 2014 18:05:54 -0700 Subject: [PATCH] add parallel in xdist --- test/integration/smoke/test_CS-18306.py | 12 +++++- tools/marvin/marvin/pytest/VM.py | 2 +- tools/pytest-xdist/xdist/dsession.py | 51 ++++++++++--------------- tools/pytest-xdist/xdist/remote.py | 11 +++++- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/test/integration/smoke/test_CS-18306.py b/test/integration/smoke/test_CS-18306.py index cb555ee1320..072da930fa4 100644 --- a/test/integration/smoke/test_CS-18306.py +++ b/test/integration/smoke/test_CS-18306.py @@ -23,8 +23,16 @@ def test_01_create_disk_offering(vm): assert vm is not None ''' -def test_a(): +class TestA: + def test_a(self): + assert True == True + + def test_b(self): + assert True == True + + +def test_aa(): assert True == True -def test_b(): +def test_bb(): assert True == True \ No newline at end of file diff --git a/tools/marvin/marvin/pytest/VM.py b/tools/marvin/marvin/pytest/VM.py index ae80518e6e4..086d8d8225e 100644 --- a/tools/marvin/marvin/pytest/VM.py +++ b/tools/marvin/marvin/pytest/VM.py @@ -81,7 +81,7 @@ def template(test_client, zone): @pytest.fixture() def vm(test_client, account, template, tiny_service_offering, zone): params = { - "displayname": "testserver", + "displayname": "!#@#@fjdkjf", "username": "root", "password": "password", "ssh_port": 22, diff --git a/tools/pytest-xdist/xdist/dsession.py b/tools/pytest-xdist/xdist/dsession.py index 8385e768a40..e5c66096c6b 100644 --- a/tools/pytest-xdist/xdist/dsession.py +++ b/tools/pytest-xdist/xdist/dsession.py @@ -61,6 +61,7 @@ class EachScheduling: class LoadScheduling: def __init__(self, numnodes, log=None): self.numnodes = numnodes + self.nodetestmap = {} self.node2pending = {} self.node2collection = {} self.nodes = [] @@ -98,24 +99,8 @@ class LoadScheduling: self.check_schedule(node, duration=duration) def check_schedule(self, node, duration=0): - if self.pending: - # how many nodes do we have? - num_nodes = len(self.node2pending) - # if our node goes below a heuristic minimum, fill it out to - # heuristic maximum - items_per_node_min = max( - 2, len(self.pending) // num_nodes // 4) - items_per_node_max = max( - 2, len(self.pending) // num_nodes // 2) - node_pending = self.node2pending[node] - if len(node_pending) < items_per_node_min: - if duration >= 0.1 and len(node_pending) >= 2: - # seems the node is doing long-running tests - # and has enough items to continue - # so let's rather wait with sending new items - return - num_send = items_per_node_max - len(node_pending) - self._send_tests(node, num_send) + if self.pending or len(self.node2collection[node] > 0): + self._send_tests(node) self.log("num items waiting for node:", len(self.pending)) #self.log("node2pending:", self.node2pending) @@ -150,22 +135,28 @@ class LoadScheduling: if not self.collection: return - # how many items per node do we have about? - items_per_node = len(self.collection) // len(self.node2pending) - # take a fraction of tests for initial distribution - node_chunksize = max(items_per_node // 4, 2) - # and initialize each node with a chunk of tests for node in self.nodes: - self._send_tests(node, node_chunksize) + self._send_tests(node) #f = open("/tmp/sent", "w") - def _send_tests(self, node, num): - tests_per_node = self.pending[:num] + def _send_tests(self, node): + if len(self.node2collection[node]) > 0: + index = self.node2collection[node].pop(0) + print index + node.send_runtest_some([index]) #print >>self.f, "sent", node, tests_per_node - if tests_per_node: - del self.pending[:num] - self.node2pending[node].extend(tests_per_node) - node.send_runtest_some(tests_per_node) + else: + index = self.pending.pop(0) + item = self.collection[index] + #how many items + indexs = [index] + for pos,name in enumerate(self.collection): + if name == item and pos != index: + idx = self.pending.pop(0) + indexs.append(idx) + self.node2collection[node] = indexs + print index + node.send_runtest_some([index]) def _check_nodes_have_same_collection(self): """ diff --git a/tools/pytest-xdist/xdist/remote.py b/tools/pytest-xdist/xdist/remote.py index a0b2cade786..dbe233f3731 100644 --- a/tools/pytest-xdist/xdist/remote.py +++ b/tools/pytest-xdist/xdist/remote.py @@ -72,9 +72,18 @@ class SlaveInteractor: nextitem=nextitem) def pytest_collection_finish(self, session): + units = [] + for item in session.items: + if item.instance is None: + units.append(item.nodeid) + else: + instance = item.instance + name = instance.__module__ + ":" + instance.__class__.__name__ + units.append(name) + self.sendevent("collectionfinish", topdir=str(session.fspath), - ids=[item.nodeid for item in session.items]) + ids=units) def pytest_runtest_logstart(self, nodeid, location): self.sendevent("logstart", nodeid=nodeid, location=location)