Added basic performance test capabilities for multiple runs.

subprojects/performance/src/integTest/groovy/org/gradle/peformance/PerformanceTest.groovy

@@ -1,7 +1,9 @@
 package org.gradle.peformance
 
+import org.gradle.integtests.fixtures.GradleDistribution
+import org.gradle.integtests.fixtures.ReleasedVersions
+import org.gradle.peformance.fixture.PerformanceTestRunner
 import spock.lang.Specification
-import org.gradle.integtests.fixtures.*
 
 /**
  * by Szczepan Faber, created at: 2/9/12
@@ -11,64 +13,20 @@ class PerformanceTest extends Specification {
     def current = new GradleDistribution()
     def previous = new ReleasedVersions(current).last
 
-    def "current release is not worse than the previous one"() {
+    def "current release is not slower than the previous one"() {
         when:
-        //actually, fails only at 16m
-        def previousExecuter = executer(previous, "small").withGradleOpts("-Xmx20m")
-        def previousResult = measure {
-            previousExecuter.run()
-        }
-
-        def currentExecuter = executer(current, "small").withGradleOpts("-Xmx20m")
-        def currentResult = measure {
-            currentExecuter.run()
-        }
-
+        def results = new PerformanceTestRunner(testProject: "small", runs: 5, warmUpRuns: 1).run()
+
         then:
-        previousResult.exception == null & currentResult.exception == null
-        previousResult.executionTime <= currentResult.executionTime
-    }
-
-    private GradleExecuter executer(BasicGradleDistribution dist, String testProjectName) {
-        def projectDir = findProjectDir(testProjectName)
-        def executer
-        if (dist instanceof GradleDistribution) {
-            executer = new GradleDistributionExecuter(GradleDistributionExecuter.Executer.forking, dist)
-        } else {
-            executer = dist.executer()
-        }
-        return executer.withArguments('-u').inDirectory(projectDir).withTasks('clean', 'build')
-    }
-
-    class MeasuredResult {
-        long executionTime
-        Exception exception
+        results.assertEveryBuildSucceeds()
+        results.assertCurrentReleaseIsNotSlower()
     }
 
-    MeasuredResult measure(Closure operation) {
-        long before = System.currentTimeMillis()
-        def out = new MeasuredResult()
-        try {
-            operation()
-        } catch (Exception e) {
-            out.exception = e
-        }
-        out.executionTime = System.currentTimeMillis() - before
-        return out
-    }
+    def "current release requires not more memory than the previous one"() {
+        when:
+        def results = new PerformanceTestRunner(testProject: "small", runs: 1, gradleOpts: ['-Xmx17m']).run()
 
-    File findProjectDir(String name) {
-        def base = "subprojects/performance/build"
-        def locations = ["$base/$name", "../../$base/$name"]
-        def dirs = locations.collect { new File(it).absoluteFile }
-        for (File dir: dirs) {
-            if (dir.isDirectory()) {
-                return dir
-            }
-        }
-        def message = "Looks like the test project '$name' was not generated.\nI've tried to find it at:\n"
-        dirs.each { message += "  $it\n" }
-        message +="Please run 'gradlew performance:$name' to generate the test project."
-        assert false: message
+        then:
+        results.assertEveryBuildSucceeds()
     }
-}
+}

subprojects/performance/src/integTest/groovy/org/gradle/peformance/fixture/MeasuredOperation.groovy

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.peformance.fixture
+
+/**
+ * by Szczepan Faber, created at: 2/10/12
+ */
+public class MeasuredOperation {
+    long executionTime
+    Exception exception
+
+    static MeasuredOperation measure(Closure operation) {
+        long before = System.currentTimeMillis()
+        def out = new MeasuredOperation()
+        try {
+            operation()
+        } catch (Exception e) {
+            out.exception = e
+        }
+        out.executionTime = System.currentTimeMillis() - before
+        return out
+    }
+}

subprojects/performance/src/integTest/groovy/org/gradle/peformance/fixture/PerformanceResults.groovy

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.peformance.fixture
+
+public class PerformanceResults {
+
+    List<MeasuredOperation> previous = new LinkedList<MeasuredOperation>()
+    List<MeasuredOperation> current = new LinkedList<MeasuredOperation>()
+
+    def clear() {
+        previous.clear();
+        current.clear();
+    }
+
+    void addResult(MeasuredOperation previous, MeasuredOperation current) {
+        this.previous.add(previous)
+        this.current.add(current)
+    }
+
+    void assertEveryBuildSucceeds() {
+        assert previous.size() == current.size()
+        def previousExceptions = previous.findAll { it.exception }
+        def currentExceptions  = previous.findAll { it.exception }
+        assert previousExceptions.isEmpty() & currentExceptions.isEmpty()
+    }
+
+    void assertCurrentReleaseIsNotSlower() {
+        assert previous.size() == current.size()
+        long averagePrevious = previous.collect { it.executionTime }.sum() / previous.size()
+        long averageCurrent  = current.collect { it.executionTime }.sum() / current.size()
+        assert averageCurrent <= averagePrevious : """Looks like the current gradle is slower than latest release.
+previous release build times: ${previous}
+current gradle build times:   ${current}
+"""
+    }
+}

subprojects/performance/src/integTest/groovy/org/gradle/peformance/fixture/PerformanceTestRunner.groovy

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.peformance.fixture
+
+import org.gradle.api.logging.Logging
+import org.gradle.integtests.fixtures.*
+
+public class PerformanceTestRunner {
+
+    private final static LOGGER = Logging.getLogger(PerformanceTestRunner.class)
+
+    def current = new GradleDistribution()
+    def previous = new ReleasedVersions(current).last
+
+    String testProject
+    int runs
+    int warmUpRuns
+    List<String> gradleOpts
+
+    def results = new PerformanceResults()
+
+    PerformanceResults run() {
+        LOGGER.lifecycle("Running performance tests for test project '{}', no. # runs: {}", testProject, runs)
+        warmUpRuns.times {
+            LOGGER.info("Executing warm-up run #${it+1}")
+            runOnce()
+        }
+        results.clear()
+        runs.times {
+            LOGGER.info("Executing test run #${it+1}")
+            runOnce()
+        }
+        results
+    }
+
+    void runOnce() {
+        def previousExecuter = executer(previous, testProject)
+        def previousResult = MeasuredOperation.measure {
+            previousExecuter.run()
+        }
+
+        def currentExecuter = executer(current, testProject)
+        def currentResult = MeasuredOperation.measure {
+            currentExecuter.run()
+        }
+
+        results.addResult(previousResult, currentResult)
+    }
+
+    GradleExecuter executer(BasicGradleDistribution dist, String testProjectName) {
+        def projectDir = new TestProjectLocator().findProjectDir(testProjectName)
+        def executer
+        if (dist instanceof GradleDistribution) {
+            executer = new GradleDistributionExecuter(GradleDistributionExecuter.Executer.forking, dist)
+        } else {
+            executer = dist.executer()
+        }
+        return executer.withArguments('-u').inDirectory(projectDir).withTasks('clean', 'build')
+    }
+}

subprojects/performance/src/integTest/groovy/org/gradle/peformance/fixture/TestProjectLocator.groovy

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.gradle.peformance.fixture
+
+/**
+ * by Szczepan Faber, created at: 2/10/12
+ */
+class TestProjectLocator {
+
+    File findProjectDir(String name) {
+        def base = "subprojects/performance/build"
+        def locations = ["$base/$name", "../../$base/$name"]
+        def dirs = locations.collect { new File(it).absoluteFile }
+        for (File dir: dirs) {
+            if (dir.isDirectory()) {
+                return dir
+            }
+        }
+        def message = "Looks like the test project '$name' was not generated.\nI've tried to find it at:\n"
+        dirs.each { message += "  $it\n" }
+        message +="Please run 'gradlew performance:$name' to generate the test project."
+        assert false: message
+    }
+}