Merge c750a20 into 04e3fcc

CHANGELOG.md

@@ -3,6 +3,9 @@
 <!-- There is always Unreleased section on the top. Subsections (Added, Changed, Fixed, Removed) should be added as needed. -->
 
 ## Unreleased
+### Added
+- `--parallel-limit` (`-l`) option of `run` command to allow limiting maximum number of tests being run simultaneously.
+
 ### Changed
 - Require PHP 7.1+ and Symfony 4 components.
 - Update to namespaced PHPUnit 7.0.

README.md

@@ -161,6 +161,7 @@ There is also a bunch of useful options for the `run` command:
 - `--server-url` - set different url of selenium server than the default (which is `http://localhost:4444/wd/hub`)
 - `--xdebug` - start Xdebug debugger on your tests. Allows you to debug tests from your IDE ([learn more about tests debugging][wiki-debugging] in our Wiki)
 - `--capability` - directly pass any extra capability to the Selenium WebDriver server ([see wiki][wiki-capabilities] for more information and examples)
+- `--parallel-limit` - limit number of testcases being executed in a parallel (default is 50)
 - `--help` - see all other options and default values
 - **adjust output levels:** by default, only the test results summary is printed to the output; the verbosity could be changed by the following:
     - `-v` - to instantly output name of failed test(s)

composer.json

@@ -36,7 +36,7 @@
         "ext-filter": "*",
         "ext-SimpleXML": "*",
         "ext-libxml": "*",
-        "phpunit/phpunit": "^7.0",
+        "phpunit/phpunit": "^7.2.1",
         "symfony/console": "^4.0",
         "symfony/process": "^4.0 !=4.0.2",
         "symfony/finder": "^4.0",

src-tests/Console/Command/Fixtures/FailingTests/expected-very-verbose-output.txt

@@ -3,6 +3,7 @@ Browser: chrome
 Environment: staging
 Path to logs: %s/logs
 Ignore delays: no
+Parallel limit: 50
 Selenium server (hub) url: %s, trying connection...OK
 Searching for testcases:
  - in directory %s/src-tests/Console/Command/Fixtures/FailingTests"

src-tests/Console/Command/Fixtures/ParallelTests/FirstTest.php

@@ -0,0 +1,16 @@
+<?php declare(strict_types=1);
+
+namespace Lmc\Steward\Console\Command\Fixtures\ParallelTests;
+
+use Lmc\Steward\Component\Legacy;
+use Lmc\Steward\Test\AbstractTestCase;
+
+class FirstTest extends AbstractTestCase
+{
+    /**
+     * @doesNotPerformAssertions
+     */
+    public function testMethod1(): void
+    {
+    }
+}

src-tests/Console/Command/Fixtures/ParallelTests/SecondTest.php

@@ -0,0 +1,15 @@
+<?php declare(strict_types=1);
+
+namespace Lmc\Steward\Console\Command\Fixtures\ParallelTests;
+
+use Lmc\Steward\Test\AbstractTestCase;
+
+class SecondTest extends AbstractTestCase
+{
+    /**
+     * @doesNotPerformAssertions
+     */
+    public function testMethod1(): void
+    {
+    }
+}

src-tests/Console/Command/Fixtures/ParallelTests/TestDependingOnFirstTest.php

@@ -0,0 +1,19 @@
+<?php declare(strict_types=1);
+
+namespace Lmc\Steward\Console\Command\Fixtures\ParallelTests;
+
+use Lmc\Steward\Test\AbstractTestCase;
+
+/**
+ * @delayAfter Lmc\Steward\Console\Command\Fixtures\ParallelTests\FirstTest
+ * @delayMinutes 0
+ */
+class TestDependingOnFirstTest extends AbstractTestCase
+{
+    /**
+     * @doesNotPerformAssertions
+     */
+    public function testMethod1(): void
+    {
+    }
+}

src-tests/Console/Command/Fixtures/ParallelTests/expected-debug-output.txt

@@ -0,0 +1,19 @@
+%A
+Parallel limit: 1
+%A
+Testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\FirstTest" is prepared to be run
+Max parallel limit reached, testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\SecondTest" is queued
+Testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\TestDependingOnFirstTest" is queued to be run 0.0 minutes after testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\FirstTest" is finished
+%A
+[%s] Execution of testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\FirstTest" started%A
+%A
+[%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\FirstTest" (result: passed, time: %f sec)
+[%s] Dequeing testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\SecondTest" which was queued because of parallel limit
+%A
+[%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\SecondTest" (result: passed, time: %f sec)
+[%s] Dequeing testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\TestDependingOnFirstTest"
+%A
+[%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\ParallelTests\TestDependingOnFirstTest" (result: passed, time: %f sec)%A
+[%s] All testcases done in %f seconds
+%A[OK] Testcases executed: 3 (passed: 3)%A
+

src-tests/Console/Command/Fixtures/SimpleTests/expected-debug-output.txt

@@ -3,6 +3,7 @@ Browser: chrome
 Environment: staging
 Path to logs: %s/logs
 Ignore delays: no
+Parallel limit: 50
 Selenium server (hub) url: %s, trying connection...OK
 Searching for testcases:
  - in directory "%s/src-tests/Console/Command/Fixtures/SimpleTests"
@@ -35,7 +36,7 @@ Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest> [%s] [WebDriver] Ex
 Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest> [%s] --- Finished execution of test "testWebpage" ---%A
 Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest> OK (1 test, 1 assertion)%A
 [%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest" (result: passed, time: %f sec)%A
-[%s] Unqueing testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest"%A
+[%s] Dequeing testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest"%A
 [%s] Execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest" started with command:
 '%s/steward/bin/phpunit-steward' '--log-junit=%s/logs/Lmc-Steward-Console-Command-Fixtures-SimpleTests-DependantTest.xml' '--configuration=%s/phpunit.xml' '%s/DependantTest.php'
 Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest> [%s] Registering test results publisher "Lmc\Steward\Publisher\XmlPublisher"%a

src-tests/Console/Command/Fixtures/SimpleTests/expected-very-verbose-output.txt

@@ -3,6 +3,7 @@ Browser: chrome
 Environment: staging
 Path to logs: %s/logs
 Ignore delays: no
+Parallel limit: 50
 Selenium server (hub) url: %s, trying connection...OK
 Searching for testcases:
  - in directory "%s/src-tests/Console/Command/Fixtures/SimpleTests"
@@ -12,7 +13,7 @@ Starting execution of testcases
 -------------------------------
 [%s] Execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest" started%A
 [%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\SimpleTest" (result: passed, time: %f sec)%A
-[%s] Unqueing testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest"%A
+[%s] Dequeing testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest"%A
 [%s] Execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest" started%A
 [%s] Finished execution of testcase "Lmc\Steward\Console\Command\Fixtures\SimpleTests\DependantTest" (result: passed, time: %f sec)
 [%s] Waiting (running: 0, queued: 0, done: 2 [passed: 2])

src-tests/Console/Command/RunCommandIntegrationTest.php

@@ -91,6 +91,26 @@ public function provideExpectedTestOutput(): array
         ];
     }
 
+    public function testShouldExecuteTestsWithParallelLimit(): void
+    {
+        $this->tester->execute(
+            [
+                'command' => $this->command->getName(),
+                'environment' => 'staging',
+                'browser' => 'chrome',
+                '--tests-dir' => __DIR__ . '/Fixtures/ParallelTests',
+                '--parallel-limit' => 1,
+            ],
+            ['verbosity' => OutputInterface::VERBOSITY_DEBUG]
+        );
+
+        $output = $this->tester->getDisplay();
+
+        $this->assertStringMatchesFormatFile(__DIR__ . '/Fixtures/ParallelTests/expected-debug-output.txt', $output);
+
+        $this->assertSame(0, $this->tester->getStatusCode());
+    }
+
     public function testShouldExecuteTestsThatDontNeedBrowser(): void
     {
         $this->tester->execute(

src-tests/Process/ExecutionLoopTest.php

@@ -7,6 +7,7 @@
 use Symfony\Component\Console\Input\StringInput;
 use Symfony\Component\Console\Output\BufferedOutput;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Process\Process;
 
 /**
  * @covers \Lmc\Steward\Process\ExecutionLoop
@@ -17,17 +18,54 @@ class ExecutionLoopTest extends TestCase
     public function shouldExecuteEmptyProcessSet(): void
     {
         $emptyProcessSet = new ProcessSet();
-        $output = new BufferedOutput(OutputInterface::VERBOSITY_DEBUG);
+        $outputBuffer = new BufferedOutput(OutputInterface::VERBOSITY_DEBUG);
 
         $loop = new ExecutionLoop(
             $emptyProcessSet,
-            new StewardStyle(new StringInput(''), $output),
+            new StewardStyle(new StringInput(''), $outputBuffer),
             new MaxTotalDelayStrategy()
         );
 
         $result = $loop->start();
+        $output = $outputBuffer->fetch();
 
-        $this->assertTrue($result);
-        $this->assertContains('[OK] Testcases executed: 0', $output->fetch());
+        $this->assertTrue($result, 'Exception loop did not finish successfully, output was: ' . "\n" . $output);
+        $this->assertContains('[OK] Testcases executed: 0', $output);
+    }
+
+    /** @test */
+    public function shouldDequeueProcessesWithoutDelayOnStartup(): void
+    {
+        $noDelayTest = new ProcessWrapper(new Process('echo NoDelay'), 'NoDelay');
+        $delayedTest = new ProcessWrapper(new Process('echo Delayed'), 'Delayed');
+        $delayedTest->setDelay('NoDelay', 0.001);
+
+        $processSet = new ProcessSet();
+        $processSet->add($noDelayTest);
+        $processSet->add($delayedTest);
+
+        // Preconditions - both processes should be queued after being added
+        $processes = $processSet->get(ProcessWrapper::PROCESS_STATUS_QUEUED);
+        $this->assertCount(2, $processes);
+
+        $outputBuffer = new BufferedOutput(OutputInterface::VERBOSITY_DEBUG);
+        $loop = new ExecutionLoop(
+            $processSet,
+            new StewardStyle(new StringInput(''), $outputBuffer),
+            new MaxTotalDelayStrategy()
+        );
+
+        $result = $loop->start();
+        $output = $outputBuffer->fetch();
+
+        $this->assertTrue($result, 'Exception loop did not finish successfully, output was: ' . "\n" . $output);
+
+        $this->assertContains('Testcase "NoDelay" is prepared to be run', $output);
+        $this->assertContains(
+            'Testcase "Delayed" is queued to be run 0.0 minutes after testcase "NoDelay" is finished',
+            $output
+        );
+
+        $this->assertContains('Dequeing testcase "Delayed"', $output);
     }
 }

src-tests/Process/ProcessSetTest.php

@@ -7,8 +7,6 @@
 use Lmc\Steward\Process\Fixtures\MockOrderStrategy;
 use Lmc\Steward\Publisher\XmlPublisher;
 use PHPUnit\Framework\TestCase;
-use Symfony\Component\Console\Output\BufferedOutput;
-use Symfony\Component\Console\Output\Output;
 use Symfony\Component\Process\Process;
 
 class ProcessSetTest extends TestCase
@@ -196,40 +194,6 @@ public function testShouldCountResultsOfDoneProcesses(): void
         );
     }
 
-    public function testShouldDequeueProcessesWithoutDelay(): void
-    {
-        $noDelayTest = new ProcessWrapper(new Process(''), 'NoDelay');
-        $delayedTest = new ProcessWrapper(new Process(''), 'Delayed');
-        $delayedTest->setDelay('NoDelay', 3.3);
-        $this->set->add($noDelayTest);
-        $this->set->add($delayedTest);
-        $outputBuffer = new BufferedOutput(Output::VERBOSITY_DEBUG);
-
-        // Preconditions - both processes should be queued after being added
-        $processes = $this->set->get(ProcessWrapper::PROCESS_STATUS_QUEUED);
-        $this->assertCount(2, $processes);
-
-        // Should Dequeue process without delay
-        $this->set->dequeueProcessesWithoutDelay($outputBuffer);
-
-        // The process without delay should be prepared now
-        $prepared = $this->set->get(ProcessWrapper::PROCESS_STATUS_PREPARED);
-        $this->assertCount(1, $prepared);
-        $this->assertSame($noDelayTest, $prepared['NoDelay']);
-
-        // The other process with delay should be kept as queued
-        $queued = $this->set->get(ProcessWrapper::PROCESS_STATUS_QUEUED);
-        $this->assertCount(1, $queued);
-        $this->assertSame($delayedTest, $queued['Delayed']);
-
-        $output = $outputBuffer->fetch();
-        $this->assertContains('Testcase "NoDelay" is prepared to be run', $output);
-        $this->assertContains(
-            'Testcase "Delayed" is queued to be run 3.3 minutes after testcase "NoDelay" is finished',
-            $output
-        );
-    }
-
     public function testShouldFailBuildingTreeIfTestHasDependencyOnNotExistingTest(): void
     {
         $process = new ProcessWrapper(new Process(''), 'Foo');

src/Console/Command/RunCommand.php

@@ -51,6 +51,7 @@ class RunCommand extends Command
     public const OPTION_FILTER = 'filter';
     public const OPTION_NO_EXIT = 'no-exit';
     public const OPTION_IGNORE_DELAYS = 'ignore-delays';
+    public const OPTION_PARALLEL_LIMIT = 'parallel-limit';
 
     /**
      * @internal
@@ -148,6 +149,13 @@ protected function configure(): void
                 'i',
                 InputOption::VALUE_NONE,
                 'Ignore delays defined between testcases'
+            )
+            ->addOption(
+                self::OPTION_PARALLEL_LIMIT,
+                'l',
+                InputOption::VALUE_REQUIRED,
+                'Number of maximum testcases being executed in a parallel',
+                50
             );
 
         $this->addUsage('staging firefox');
@@ -204,6 +212,13 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
         $seleniumAdapter = $this->getSeleniumAdapter($input->getOption(self::OPTION_SERVER_URL));
         $input->setOption(self::OPTION_SERVER_URL, $seleniumAdapter->getServerUrl());
 
+        // Make sure parallel-limit is greater than 0
+        $parallelLimit = (int) $input->getOption(self::OPTION_PARALLEL_LIMIT);
+        if ($parallelLimit === 0) {
+            throw new \RuntimeException('Parallel limit must be a whole number greater than 0');
+        }
+        $input->setOption(self::OPTION_PARALLEL_LIMIT, $parallelLimit);
+
         $this->getDispatcher()->dispatch(
             CommandEvents::RUN_TESTS_INIT,
             new ExtendedConsoleEvent($this, $input, $output)
@@ -216,6 +231,9 @@ protected function initialize(InputInterface $input, OutputInterface $output): v
             $output->writeln(
                 sprintf('Ignore delays: %s', ($input->getOption(self::OPTION_IGNORE_DELAYS)) ? 'yes' : 'no')
             );
+            $output->writeln(
+                sprintf('Parallel limit: %d', $input->getOption(self::OPTION_PARALLEL_LIMIT))
+            );
         }
     }
 
@@ -263,7 +281,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return 1;
         }
 
-        $executionLoop = new ExecutionLoop($processSet, $this->io, new MaxTotalDelayStrategy());
+        $maxParallelLimit = $input->getOption(self::OPTION_PARALLEL_LIMIT);
+
+        $executionLoop = new ExecutionLoop($processSet, $this->io, new MaxTotalDelayStrategy(), $maxParallelLimit);
 
         $allTestsPassed = $executionLoop->start();