From f26498233fc910a348b06e8e7564544c9a508464 Mon Sep 17 00:00:00 2001 From: dsankouski Date: Sun, 12 Dec 2021 18:09:16 +0300 Subject: [PATCH] Enable ParallelSuiteTest (#2687) * Enable ParallelSuiteTest Presumably, test was disabled to get rid of distraction during development and was forgotten. Test was broken and commented out temporary back in 2011 in cc950e9c5f035c275e46d2e58901784e03b6959f commit, named `Temp commit`. Commit was introduced in `AllDynamic` branch Test is green both on the base commit of AllDynamic branch (10a202a9e96a8067720d9e2a301ee6b67f5df5ed) as well as on branch merge point (b251fce83189795bb9eaa8b0d2d1505b935f4c24) Done: - Add test: child suite should obey threadCount parameter - Move private stuff to bottom, add more details in error message * Fix: setup parallel option for all suites, including child * Fix populateSuiteGraph method: add parent nodes, too Before this, populateSuiteGraph method was ignoring any suite runner with child suites. This leads to parallel suites not being executed with `randomize suite` option enabled The method now add all nodes, including parents into the graph. * Adjust expected results for suitesShouldRunInParallel4 This test implied a requirement to exclude duplicating suite files from run. For example, for suite graph like: suite-parallel-1.xml suite-parallel-2.xml suite-parallel-2-1.xml suite-parallel-2-2.xml | | | | ---------- ---------| v v v suite-parallel-2-2-1.xml suite-parallel-2-1.xml suite-parallel-2-2.xml | | v suite-parallel-2-2-1.xml a test expected 5 suites and 5 threads. However, this test was excluded from TestNG unit tests since 2011, and broken somewhere later on the timeline. Dropping mentioned requirement (i.e. leaving things as it is now), since there's no complains on current behaviour by adjusting test expected results, to 8 expected suites and 8 threads. * review fixes: - revert suite order - gradle autostyle apply - set parallel mode recursively Co-authored-by: sankouski-dzmitry Co-authored-by: Dzmitry Sankouski --- .../src/main/java/org/testng/TestNG.java | 24 +-- .../java/test/thread/ParallelSuiteTest.java | 142 +++++++++++++----- .../inherit-thread-count-child.yaml | 7 + .../inherit-thread-count-parent.yaml | 5 + .../simple-suite-parallel-0.xml} | 6 +- .../simple-suite-parallel-1.xml} | 0 .../simple-suite-parallel-2.xml} | 0 testng-core/src/test/resources/testng.xml | 2 - testng-core/src/test/resources/testng.yaml | 1 + 9 files changed, 132 insertions(+), 55 deletions(-) create mode 100644 testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml create mode 100644 testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml rename testng-core/src/test/resources/{suite-parallel-0.xml => parallel-suites/simple-suite-parallel-0.xml} (69%) rename testng-core/src/test/resources/{suite-parallel-1.xml => parallel-suites/simple-suite-parallel-1.xml} (100%) rename testng-core/src/test/resources/{suite-parallel-2.xml => parallel-suites/simple-suite-parallel-2.xml} (100%) diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index e9e1ab59b4..1324909e9a 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -344,12 +344,7 @@ private Collection parseSuite(String suitePath) { private Collection processCommandLineArgs(Collection allSuites) { Collection result = new ArrayList<>(); for (XmlSuite s : allSuites) { - if (this.m_parallelMode != null) { - s.setParallel(this.m_parallelMode); - } - if (this.m_threadCount > 0) { - s.setThreadCount(this.m_threadCount); - } + processParallelModeCommandLineArgs(s); if (m_testNames == null) { result.add(s); continue; @@ -366,6 +361,18 @@ private Collection processCommandLineArgs(Collection allSuit return result; } + private void processParallelModeCommandLineArgs(XmlSuite suite) { + if (this.m_parallelMode != null) { + suite.setParallel(this.m_parallelMode); + } + if (this.m_threadCount > 0) { + suite.setThreadCount(this.m_threadCount); + } + if (suite.getChildSuites() != null) { + suite.getChildSuites().forEach(this::processParallelModeCommandLineArgs); + } + } + public void initializeSuitesAndJarFile() { // The IntelliJ plug-in might have invoked this method already so don't initialize suites twice. if (isSuiteInitialized) { @@ -1256,9 +1263,8 @@ private void populateSuiteGraph( SuiteRunnerMap suiteRunnerMap, XmlSuite xmlSuite) { ISuite parentSuiteRunner = suiteRunnerMap.get(xmlSuite); - if (xmlSuite.getChildSuites().isEmpty()) { - suiteGraph.addNode(parentSuiteRunner); - } else { + suiteGraph.addNode(parentSuiteRunner); + if (!xmlSuite.getChildSuites().isEmpty()) { for (XmlSuite childSuite : xmlSuite.getChildSuites()) { suiteGraph.addEdge(0, parentSuiteRunner, suiteRunnerMap.get(childSuite)); populateSuiteGraph(suiteGraph, suiteRunnerMap, childSuite); diff --git a/testng-core/src/test/java/test/thread/ParallelSuiteTest.java b/testng-core/src/test/java/test/thread/ParallelSuiteTest.java index 2a8f55671b..dcbe8f721c 100644 --- a/testng-core/src/test/java/test/thread/ParallelSuiteTest.java +++ b/testng-core/src/test/java/test/thread/ParallelSuiteTest.java @@ -9,6 +9,7 @@ import org.testng.TestListenerAdapter; import org.testng.TestNG; import org.testng.annotations.Test; +import org.testng.xml.XmlSuite; import test.SimpleBaseTest; public class ParallelSuiteTest extends SimpleBaseTest { @@ -21,12 +22,18 @@ public void suitesShouldRunInParallel1() { 2, null, Arrays.asList( - getPathToResource("suite-parallel-1.xml"), getPathToResource("suite-parallel-2.xml"))); + getPathToParallelResource("simple-suite-parallel-1.xml"), + getPathToParallelResource("simple-suite-parallel-2.xml"))); } @Test public void suitesShouldRunInParallel2() { - runTest(5, 3, 3, null, Collections.singletonList(getPathToResource("suite-parallel-0.xml"))); + runTest( + 5, + 3, + 3, + null, + Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml"))); } @Test(description = "Number of threads (2) is less than number of suites (3)") @@ -35,7 +42,8 @@ public void suitesShouldRunInParallel3() { TestListenerAdapter tla = new TestListenerAdapter(); TestNG tng = create(); tng.setSuiteThreadPoolSize(SUITE_THREAD_POOL_SIZE); - tng.setTestSuites(Collections.singletonList(getPathToResource("suite-parallel-0.xml"))); + tng.setTestSuites( + Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml"))); tng.addListener((ITestNGListener) tla); BaseThreadTest.initThreadLog(); @@ -43,46 +51,19 @@ public void suitesShouldRunInParallel3() { Assert.assertEquals(BaseThreadTest.getThreadCount(), SUITE_THREAD_POOL_SIZE); } - private void runTest( - int suiteThreadPoolSize, - int expectedThreadCount, - int expectedSuiteCount, - Boolean randomizeSuites, - List paths) { - TestListenerAdapter tla = new TestListenerAdapter(); - TestNG tng = create(); - tng.setSuiteThreadPoolSize(suiteThreadPoolSize); - tng.setTestSuites(paths); - tng.addListener((ITestNGListener) tla); - if (null != randomizeSuites) { - tng.setRandomizeSuites(randomizeSuites); - } - - BaseThreadTest.initThreadLog(); - tng.run(); - - Assert.assertEquals( - BaseThreadTest.getThreadCount(), - expectedThreadCount, - "Thread count expected:" - + expectedThreadCount - + " actual:" - + BaseThreadTest.getThreadCount()); - Assert.assertEquals(BaseThreadTest.getSuitesMap().keySet().size(), expectedSuiteCount); - } - @Test public void suitesShouldRunInParallel4() { + final int TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES = 8; runTest( 10, - 5, - 5, + TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES, + TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES, null, Arrays.asList( - getPathToResource("parallel-suites/suite-parallel-1.xml"), - getPathToResource("parallel-suites/suite-parallel-2.xml"), - getPathToResource("parallel-suites/suite-parallel-2-1.xml"), - getPathToResource("parallel-suites/suite-parallel-2-2.xml"))); + getPathToParallelResource("suite-parallel-1.xml"), + getPathToParallelResource("suite-parallel-2.xml"), + getPathToParallelResource("suite-parallel-2-1.xml"), + getPathToParallelResource("suite-parallel-2-2.xml"))); } @Test @@ -92,7 +73,7 @@ public void suitesShouldRunInParallel5() { 5, 7, null, - Collections.singletonList(getPathToResource("parallel-suites/suite-parallel-0.xml"))); + Collections.singletonList(getPathToParallelResource("suite-parallel-0.xml"))); } @Test(description = "Number of threads (2) is less than level of suites (3)") @@ -102,7 +83,7 @@ public void suitesShouldRunInParallel6() { 2, 7, null, - Collections.singletonList(getPathToResource("parallel-suites/suite-parallel-0.xml"))); + Collections.singletonList(getPathToParallelResource("suite-parallel-0.xml"))); } @Test( @@ -112,7 +93,8 @@ public void suitesShouldRunInParallel6() { public void suitesShouldRunInOrder() { TestListenerAdapter tla = new TestListenerAdapter(); TestNG tng = create(); - tng.setTestSuites(Collections.singletonList(getPathToResource("suite-parallel-0.xml"))); + tng.setTestSuites( + Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml"))); tng.addListener((ITestNGListener) tla); BaseThreadTest.initThreadLog(); tng.run(); @@ -134,6 +116,84 @@ public void suitesShouldRunInOrder() { @Test(description = "Number of threads (1) is less than number of levels of suites (2)") public void suitesShouldRun1() { - runTest(1, 1, 3, true, Collections.singletonList(getPathToResource("suite-parallel-0.xml"))); + runTest( + 1, + 1, + 3, + true, + Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml"))); + } + + @Test(description = "Child suite should obey threadCount parameter") + public void childSuiteObeyParentThreadCount() { + /* parent suite has no tests, so only child suite counts */ + final int EXPECTED_SUITE_COUNT = 1; + runTest( + 1, + 2, + XmlSuite.ParallelMode.CLASSES, + 2, + EXPECTED_SUITE_COUNT, + null, + Arrays.asList(getPathToParallelResource("inherit-thread-count-parent.yaml"))); + } + + private void runTest( + int suiteThreadPoolSize, + int expectedThreadCount, + int expectedSuiteCount, + Boolean randomizeSuites, + List paths) { + runTest( + suiteThreadPoolSize, + null, + null, + expectedThreadCount, + expectedSuiteCount, + randomizeSuites, + paths); + } + + private void runTest( + int suiteThreadPoolSize, + Integer threadCount, + XmlSuite.ParallelMode parallelMode, + int expectedThreadCount, + int expectedSuiteCount, + Boolean randomizeSuites, + List paths) { + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG tng = create(); + tng.setSuiteThreadPoolSize(suiteThreadPoolSize); + if (threadCount != null) { + tng.setThreadCount(threadCount); + } + if (parallelMode != null) { + tng.setParallel(parallelMode); + } + tng.setTestSuites(paths); + tng.addListener((ITestNGListener) tla); + if (null != randomizeSuites) { + tng.setRandomizeSuites(randomizeSuites); + } + + BaseThreadTest.initThreadLog(); + tng.run(); + + Assert.assertEquals( + BaseThreadTest.getThreadCount(), + expectedThreadCount, + "Thread count expected:" + + expectedThreadCount + + " actual:" + + BaseThreadTest.getThreadCount()); + Assert.assertEquals( + BaseThreadTest.getSuitesMap().keySet().size(), + expectedSuiteCount, + "Suite count is incorrect"); + } + + private static String getPathToParallelResource(String resourceName) { + return getPathToResource(String.format("parallel-suites/%s", resourceName)); } } diff --git a/testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml new file mode 100644 index 0000000000..1cc8ef4218 --- /dev/null +++ b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml @@ -0,0 +1,7 @@ +name: Child suite + +tests: + - name: Tests + classes: + - test.thread.Sample2 + - test.thread.Sample1 diff --git a/testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml new file mode 100644 index 0000000000..88d55ee407 --- /dev/null +++ b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml @@ -0,0 +1,5 @@ +name: Parent suite + +suiteFiles: + - ./inherit-thread-count-child.yaml + diff --git a/testng-core/src/test/resources/suite-parallel-0.xml b/testng-core/src/test/resources/parallel-suites/simple-suite-parallel-0.xml similarity index 69% rename from testng-core/src/test/resources/suite-parallel-0.xml rename to testng-core/src/test/resources/parallel-suites/simple-suite-parallel-0.xml index e48c84bc0b..ee42982c80 100644 --- a/testng-core/src/test/resources/suite-parallel-0.xml +++ b/testng-core/src/test/resources/parallel-suites/simple-suite-parallel-0.xml @@ -3,8 +3,8 @@ - - + + @@ -12,4 +12,4 @@ - \ No newline at end of file + diff --git a/testng-core/src/test/resources/suite-parallel-1.xml b/testng-core/src/test/resources/parallel-suites/simple-suite-parallel-1.xml similarity index 100% rename from testng-core/src/test/resources/suite-parallel-1.xml rename to testng-core/src/test/resources/parallel-suites/simple-suite-parallel-1.xml diff --git a/testng-core/src/test/resources/suite-parallel-2.xml b/testng-core/src/test/resources/parallel-suites/simple-suite-parallel-2.xml similarity index 100% rename from testng-core/src/test/resources/suite-parallel-2.xml rename to testng-core/src/test/resources/parallel-suites/simple-suite-parallel-2.xml diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index d2a428da7e..f9b90ca4c4 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -109,9 +109,7 @@ - diff --git a/testng-core/src/test/resources/testng.yaml b/testng-core/src/test/resources/testng.yaml index c070a9d684..cbf371e098 100644 --- a/testng-core/src/test/resources/testng.yaml +++ b/testng-core/src/test/resources/testng.yaml @@ -50,6 +50,7 @@ tests: - test.thread.ThreadPoolSizeTest - test.thread.SequentialTest - test.thread.ParallelTestTest + - test.thread.ParallelSuiteTest - test.thread.FactoryTest - test.thread.DataProviderThreadPoolSizeTest - test.thread.MultiThreadedDependentTest