New performance tests compare the runners for now

giorgiosironi · Feb 17, 2013 · 5150ddc · 5150ddc
1 parent f28c4dd
commit 5150ddc
Showing 1 changed file with 56 additions and 8 deletions.
diff --git a/functional/PerformanceTest.php b/functional/PerformanceTest.php
@@ -4,6 +4,19 @@ class PerformanceTest extends FunctionalTestBase
 {
     protected static $testTime = '/Time: (([0-9]+)(?:[.][0-9]+)?)/';
 
+    public function setUp()
+    {
+        parent::setUp();
+        foreach (glob(FIXTURES . '/small-tests/FastUnit*Test.php') as $generatedFile) {
+            unlink($generatedFile);
+        }
+    }
+
+    private function createSmallTests($number)
+    {
+        exec("php {$this->path}/generate.php $number", $output);
+    }
+
     public function testRunningSuitesInParallelIsNotSlower()
     {
         list($stdTime, $paraTime, $msg) = $this->getExecTimes(
@@ -13,27 +26,62 @@ public function testRunningSuitesInParallelIsNotSlower()
         $this->assertTrue($paraTime <= $stdTime, $msg);
     }
 
-    public function testRunningSuitesWithLongBootstrapsInParallelIsNotSlower()
+    public function testRunningSuitesWithLongBootstrapsInFasterWithTheWrapperRunner()
+    {
+        $this->bootstrap = dirname(FIXTURES) . DS . 'slow_bootstrap.php';
+        list($paraTime, $wrapperParaTime, $msg) = $this->getExecTimes(
+            $output = $this->getParaTestOutput(false, array(
+                'runner' => 'Runner',
+                'processes' => 2
+            )),
+            $output = $this->getParaTestOutput(false, array(
+                'runner' => 'WrapperRunner',
+                'processes' => 2
+            ))
+        );
+        $this->assertTrue($wrapperParaTime <= $paraTime, $msg);
+    }
+
+    public function testRunningSuitesWithLongBootstrapsIsReliablyFasterThanVanillaPhpunit()
     {
+        $this->markTestIncomplete("Currently the execution times are comparable.");
         $this->bootstrap = dirname(FIXTURES) . DS . 'slow_bootstrap.php';
         list($stdTime, $paraTime, $msg) = $this->getExecTimes(
             $this->getPhpunitOutput(),
-            $output = $this->getParaTestOutput(false, array(
+            $this->getParaTestOutput(false, array(
                 'runner' => 'WrapperRunner',
                 'processes' => 2
-            )));
-        var_dump($stdTime, $paraTime);
+            ))
+        );
         $this->assertTrue($paraTime <= $stdTime, $msg);
     }
 
-    public function testRunningLotsOfShortTestsIsNotSlower()
+    public function testRunningLotsOfShortTestsIsFasterWithTheWrapperRunner()
+    {
+        $this->path = FIXTURES . DS . 'small-tests';
+        $this->createSmallTests(500);
+        list($paraTime, $wrapperParaTime, $msg) = $this->getExecTimes(
+            $this->getParaTestOutput(false, array(
+                'runner' => 'Runner',
+            )),
+            $this->getParaTestOutput(false, array(
+                'runner' => 'WrapperRunner',
+            ))
+        );
+        $this->assertTrue($wrapperParaTime <= $paraTime, $msg);
+    }
+
+    public function testRunningLotsOfShortTestsIsReliablyFasterThanWithVanillaPhpunit()
     {
+        $this->markTestIncomplete("Currently the execution times are comparable.");
         $this->path = FIXTURES . DS . 'small-tests';
-        exec("php {$this->path}/generate.php 100", $output);
+        $this->createSmallTests(500);
         list($stdTime, $paraTime, $msg) = $this->getExecTimes(
             $this->getPhpunitOutput(),
-            $this->getParaTestOutput(false, array('runner' => 'WrapperRunner')));
-        var_dump($stdTime, $paraTime);
+            $this->getParaTestOutput(false, array(
+                'runner' => 'WrapperRunner',
+            ))
+        );
         $this->assertTrue($paraTime <= $stdTime, $msg);
     }