Skip to content
Permalink
Browse files

Rerun distributed performance test in RERUNNER step (#8801)

After the improvement of automatically rerunning and tagging, we want to manage performance test in the same way:

- Only run each performance test scenario once.
- If it fails, `GRADLE_RERUNNER` will kick in and rerun the failed scenario. The good thing is that it might be scheduled to another build agent, which mitigates the effect of bad agent.

This PR does:
- Remove all `Retry` from performance tests.
- Add `GRADLE_RERUNNER` to performance tests and refactor some code.
- Add tests for `PerformanceTest`.
- Since `GRADLE_RERUNNER` depends on reading of test binary result, write binary test result file in `RerunableDistributedPerformanceTest`.
  • Loading branch information...
blindpirate committed Apr 4, 2019
1 parent ad1c8c6 commit 27529841ebd44f35d69b6c2abce65b8d2f5f07c6
Showing with 591 additions and 163 deletions.
  1. +1 −0 .teamcity/Gradle_Check/configurations/GradleBuildConfigurationDefaults.kt
  2. +0 −49 .teamcity/Gradle_Check/configurations/PerformanceTest.kt
  3. +67 −0 .teamcity/Gradle_Check/configurations/PerformanceTestCoordinator.kt
  4. +4 −4 .teamcity/Gradle_Check/model/CIBuildModel.kt
  5. +2 −2 .teamcity/Gradle_Check/projects/StageProject.kt
  6. +1 −1 .teamcity/Gradle_Util_Performance/buildTypes/AdHocPerformanceScenarioLinux.kt
  7. +1 −1 .teamcity/Gradle_Util_Performance/buildTypes/AdHocPerformanceTestCoordinatorLinux.kt
  8. +7 −8 .teamcity/common/performance-test-extensions.kt
  9. +1 −1 .teamcityTest/Gradle_Check_Tests/ApplyDefaultConfigurationTest.kt
  10. +108 −0 .teamcityTest/Gradle_Check_Tests/PerformanceTestBuildTypeTest.kt
  11. +31 −3 ...dSrc/subprojects/performance/src/main/groovy/org/gradle/testing/DistributedPerformanceTest.groovy
  12. +19 −35 ...subprojects/performance/src/main/groovy/org/gradle/testing/ReportGenerationPerformanceTest.groovy
  13. +101 −0 ...rojects/performance/src/main/groovy/org/gradle/testing/RerunableDistributedPerformanceTest.groovy
  14. +35 −0 buildSrc/subprojects/performance/src/main/groovy/org/gradle/testing/ScenarioBuildResultData.groovy
  15. +93 −0 ...cts/performance/src/test/groovy/org/gradle/testing/RerunableDistributedPerformanceTestTest.groovy
  16. +13 −8 buildSrc/subprojects/plugins/src/main/kotlin/org/gradle/plugins/performance/PerformanceTestPlugin.kt
  17. +0 −8 ...bprojects/profiling/src/main/kotlin/org/gradle/gradlebuild/profiling/buildscan/BuildScanPlugin.kt
  18. +0 −5 ...rformance-testing/src/main/groovy/org/gradle/performance/AbstractCrossBuildPerformanceTest.groovy
  19. +0 −5 ...ormance-testing/src/main/groovy/org/gradle/performance/AbstractCrossVersionPerformanceTest.groovy
  20. +2 −2 ...erformance-testing/src/main/groovy/org/gradle/performance/results/AbstractTablePageGenerator.java
  21. +9 −5 ...rformance-testing/src/main/groovy/org/gradle/performance/results/FlakinessIndexPageGenerator.java
  22. +83 −12 ...ternal-performance-testing/src/main/groovy/org/gradle/performance/results/IndexPageGenerator.java
  23. +12 −10 ...performance-testing/src/main/groovy/org/gradle/performance/results/ScenarioBuildResultData.groovy
  24. +1 −4 ...formanceTest/groovy/org/gradle/performance/regression/inception/GradleBuildPerformanceTest.groovy
@@ -124,6 +124,7 @@ fun BaseGradleBuildType.gradleRerunnerStep(model: CIBuildModel, gradleTasks: Str
"-PteamCityBuildId=%teamcity.build.id%" +
buildScanTags.map { configurations.buildScanTag(it) } +
"-PonlyPreviousFailedTestClasses=true" +
"-Dscan.tag.RERUN_TESTS" +
"-PgithubToken=%github.ci.oauth.token%"
).joinToString(separator = " ")
}

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,67 @@
package configurations

import common.Os
import common.applyPerformanceTestSettings
import common.buildToolGradleParameters
import common.checkCleanM2
import common.distributedPerformanceTestParameters
import common.gradleWrapper
import common.performanceTestCommandLine
import jetbrains.buildServer.configs.kotlin.v2018_2.AbsoluteId
import jetbrains.buildServer.configs.kotlin.v2018_2.BuildStep
import jetbrains.buildServer.configs.kotlin.v2018_2.BuildStep.ExecutionMode
import jetbrains.buildServer.configs.kotlin.v2018_2.BuildSteps
import model.CIBuildModel
import model.PerformanceTestType
import model.Stage

class PerformanceTestCoordinator(model: CIBuildModel, type: PerformanceTestType, stage: Stage) : BaseGradleBuildType(model, stage = stage, init = {
uuid = type.asId(model)
id = AbsoluteId(uuid)
name = "Performance ${type.name.capitalize()} Coordinator - Linux"

applyPerformanceTestSettings(timeout = type.timeout)

if (type == PerformanceTestType.test) {
features {
publishBuildStatusToGithub()
}
}

params {
param("performance.baselines", type.defaultBaselines)
}

fun BuildSteps.runner(runnerName: String, runnerTasks: String, extraParameters: String = "", runnerExecutionMode: BuildStep.ExecutionMode = ExecutionMode.DEFAULT) {
gradleWrapper {
name = runnerName
tasks = ""
executionMode = runnerExecutionMode
gradleParams = (performanceTestCommandLine(task = runnerTasks, baselines = "%performance.baselines%", extraParameters = type.extraParameters)
+ buildToolGradleParameters(isContinue = false)
+ distributedPerformanceTestParameters(IndividualPerformanceScenarioWorkers(model).id.toString())
+ listOf(buildScanTag("PerformanceTest"))
+ model.parentBuildCache.gradleParameters(Os.linux)
+ extraParameters
).joinToString(separator = " ")
}
}

steps {
runner("GRADLE_RUNNER", "clean distributed${type.taskId}s")
checkCleanM2()
if (type.hasRerunner) {
val rerunnerParameters = listOf(
"-PteamCityBuildId=%teamcity.build.id%",
"-PonlyPreviousFailedTestClasses=true",
"-PgithubToken=%github.ci.oauth.token%",
"-Dscan.tag.RERUN_TESTS")
runner("GRADLE_RERUNNER", "tagBuild distributed${type.taskId}s", rerunnerParameters.joinToString(" "), ExecutionMode.RUN_ON_FAILURE)
} else {
tagBuild(model, true)
}
}

applyDefaultDependencies(model, this, true)
})

@@ -257,13 +257,13 @@ enum class TestType(val unitTests: Boolean = true, val functionalTests: Boolean
forceRealizeDependencyManagement(false, true, false)
}

enum class PerformanceTestType(val taskId: String, val timeout : Int, val defaultBaselines: String = "", val extraParameters : String = "") {
enum class PerformanceTestType(val taskId: String, val timeout: Int, val defaultBaselines: String = "", val extraParameters: String = "", val hasRerunner: Boolean = true) {
test("PerformanceTest", 420, "defaults"),
experiment("PerformanceExperiment", 420, "defaults"),
flakinessDetection("FlakinessDetection", 420, "flakiness-detection-commit"),
historical("FullPerformanceTest", 2280, "2.14.1,3.5.1,4.0,last", "--checks none");
flakinessDetection("FlakinessDetection", 420, "flakiness-detection-commit", hasRerunner = false),
historical("FullPerformanceTest", 2280, "2.14.1,3.5.1,4.0,last", "--checks none", hasRerunner = false);

fun asId(model : CIBuildModel): String {
fun asId(model: CIBuildModel): String {
return "${model.projectPrefix}Performance${name.capitalize()}Coordinator"
}
}
@@ -1,7 +1,7 @@
package projects

import configurations.FunctionalTest
import configurations.PerformanceTest
import configurations.PerformanceTestCoordinator
import configurations.SanityCheck
import configurations.buildReportTab
import jetbrains.buildServer.configs.kotlin.v2018_2.AbsoluteId
@@ -36,7 +36,7 @@ class StageProject(model: CIBuildModel, stage: Stage, containsDeferredTests: Boo
specificBuildTypes.forEach { buildType(it) }

stage.performanceTests.forEach {
buildType(PerformanceTest(model, it, stage))
buildType(PerformanceTestCoordinator(model, it, stage))
}

stage.functionalTests.forEach { testCoverage ->
@@ -42,7 +42,7 @@ object AdHocPerformanceScenarioLinux : BuildType({
gradleWrapper {
name = "GRADLE_RUNNER"
gradleParams = (
performanceTestCommandLine("%templates% performance:performanceAdHocTest", "%baselines%",
performanceTestCommandLine("clean %templates% performance:performanceAdHocTest", "%baselines%",
"""--scenarios "%scenario%" --warmups %warmups% --runs %runs% --checks %checks% --channel %channel% %flamegraphs% %additional.gradle.parameters%""")
+ buildToolGradleParameters()
+ builtInRemoteBuildCacheNode.gradleParameters(Os.linux)
@@ -30,7 +30,7 @@ object AdHocPerformanceTestCoordinatorLinux : BuildType({
tasks = ""
gradleParams = (
buildToolGradleParameters(isContinue = false)
+ performanceTestCommandLine(task = "distributedPerformanceTests", baselines = "%performance.baselines%")
+ performanceTestCommandLine(task = "clean distributedPerformanceTests", baselines = "%performance.baselines%")
+ distributedPerformanceTestParameters("Gradle_Check_IndividualPerformanceScenarioWorkersLinux")
+ builtInRemoteBuildCacheNode.gradleParameters(Os.linux)
).joinToString(separator = " ")
@@ -35,20 +35,19 @@ fun BuildType.applyPerformanceTestSettings(os: Os = Os.linux, timeout: Int = 30)
param("env.BUILD_BRANCH", "%teamcity.build.branch%")
param("performance.db.url", "jdbc:h2:ssl://dev61.gradle.org:9092")
param("performance.db.username", "tcagent")
param("TC_USERNAME", "TeamcityRestBot")
}
}

fun performanceTestCommandLine(task: String, baselines: String, extraParameters: String = "", testJavaHome: String = coordinatorPerformanceTestJavaHome) = listOf(
"clean $task --baselines $baselines $extraParameters",
"-x prepareSamples",
"-Porg.gradle.performance.branchName=%teamcity.build.branch%",
"-Porg.gradle.performance.db.url=%performance.db.url% -Porg.gradle.performance.db.username=%performance.db.username% -Porg.gradle.performance.db.password=%performance.db.password.tcagent%",
"-PteamCityUsername=%TC_USERNAME% -PteamCityPassword=%teamcity.password.restbot%",
"-PtestJavaHome=$testJavaHome"
"$task --baselines $baselines $extraParameters",
"-x prepareSamples",
"-Porg.gradle.performance.branchName=%teamcity.build.branch%",
"-Porg.gradle.performance.db.url=%performance.db.url% -Porg.gradle.performance.db.username=%performance.db.username% -Porg.gradle.performance.db.password=%performance.db.password.tcagent%",
"-PteamCityUsername=%teamcity.username.restbot% -PteamCityPassword=%teamcity.password.restbot%",
"-PtestJavaHome=$testJavaHome"
)

fun distributedPerformanceTestParameters(workerId: String = "Gradle_Check_IndividualPerformanceScenarioWorkersLinux") = listOf(
"-Porg.gradle.performance.buildTypeId=${workerId} -Porg.gradle.performance.workerTestTaskName=fullPerformanceTest -Porg.gradle.performance.coordinatorBuildId=%teamcity.build.id%"
"-Porg.gradle.performance.buildTypeId=${workerId} -Porg.gradle.performance.workerTestTaskName=fullPerformanceTest -Porg.gradle.performance.coordinatorBuildId=%teamcity.build.id%"
)

@@ -113,7 +113,7 @@ class ApplyDefaultConfigurationTest {
assertEquals(BuildStep.ExecutionMode.RUN_ON_FAILURE, getGradleStep("GRADLE_RERUNNER").executionMode)

assertEquals(expectedRunnerParam(expectedDaemonParam, extraParameters), getGradleStep("GRADLE_RUNNER").gradleParams)
assertEquals(expectedRunnerParam(expectedDaemonParam, extraParameters) + " -PonlyPreviousFailedTestClasses=true -PgithubToken=%github.ci.oauth.token%", getGradleStep("GRADLE_RERUNNER").gradleParams)
assertEquals(expectedRunnerParam(expectedDaemonParam, extraParameters) + " -PonlyPreviousFailedTestClasses=true -Dscan.tag.RERUN_TESTS -PgithubToken=%github.ci.oauth.token%", getGradleStep("GRADLE_RERUNNER").gradleParams)
assertEquals("clean myTask", getGradleStep("GRADLE_RUNNER").tasks)
assertEquals("myTask tagBuild", getGradleStep("GRADLE_RERUNNER").tasks)
}
@@ -0,0 +1,108 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import common.JvmVendor
import common.JvmVersion
import common.Os
import configurations.BaseGradleBuildType
import configurations.PerformanceTestCoordinator
import jetbrains.buildServer.configs.kotlin.v2018_2.BuildStep
import jetbrains.buildServer.configs.kotlin.v2018_2.buildSteps.GradleBuildStep
import model.PerformanceTestType
import model.SpecificBuild
import model.Stage
import model.StageNames
import model.TestCoverage
import model.TestType
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test

class PerformanceTestBuildTypeTest {
private
val buildModel = model.CIBuildModel(buildScanTags = listOf("Check"))

@Test
fun `create correct PerformanceTest build type`() {
val performanceTest = PerformanceTestCoordinator(buildModel, PerformanceTestType.test, Stage(StageNames.READY_FOR_MERGE,
specificBuilds = listOf(
SpecificBuild.BuildDistributions,
SpecificBuild.Gradleception,
SpecificBuild.SmokeTests),
functionalTests = listOf(
TestCoverage(1, TestType.platform, Os.linux, JvmVersion.java8),
TestCoverage(2, TestType.platform, Os.windows, JvmVersion.java11, vendor = JvmVendor.openjdk)),
performanceTests = listOf(PerformanceTestType.test),
omitsSlowProjects = true))

assertEquals(listOf(
"GRADLE_RUNNER",
"CHECK_CLEAN_M2",
"GRADLE_RERUNNER"
), performanceTest.steps.items.map(BuildStep::name))

val expectedRunnerParams = listOf(
"--baselines",
"%performance.baselines%",
"",
"-x",
"prepareSamples",
"-Porg.gradle.performance.branchName=%teamcity.build.branch%",
"-Porg.gradle.performance.db.url=%performance.db.url%",
"-Porg.gradle.performance.db.username=%performance.db.username%",
"-Porg.gradle.performance.db.password=%performance.db.password.tcagent%",
"-PteamCityUsername=%teamcity.username.restbot%",
"-PteamCityPassword=%teamcity.password.restbot%",
"-PtestJavaHome=%linux.java8.oracle.64bit%",
"-PmaxParallelForks=%maxParallelForks%",
"-s",
"--daemon",
"",
"-I",
"\"%teamcity.build.checkoutDir%/gradle/init-scripts/build-scan.init.gradle.kts\"",
"-Dorg.gradle.internal.tasks.createops",
"-Dorg.gradle.internal.plugins.portal.url.override=%gradle.plugins.portal.url%",
"-Porg.gradle.performance.buildTypeId=Gradle_Check_IndividualPerformanceScenarioWorkersLinux",
"-Porg.gradle.performance.workerTestTaskName=fullPerformanceTest",
"-Porg.gradle.performance.coordinatorBuildId=%teamcity.build.id%",
"\"-Dscan.tag.PerformanceTest\"",
"--build-cache",
"\"-Dgradle.cache.remote.url=%gradle.cache.remote.url%\"",
"\"-Dgradle.cache.remote.username=%gradle.cache.remote.username%\"",
"\"-Dgradle.cache.remote.password=%gradle.cache.remote.password%\""
)

assertEquals(
(listOf("clean", "distributedPerformanceTests") + expectedRunnerParams).joinToString(" "),
performanceTest.getGradleStep("GRADLE_RUNNER").gradleParams!!.trim()
)
assertEquals(BuildStep.ExecutionMode.DEFAULT, performanceTest.getGradleStep("GRADLE_RUNNER").executionMode)

assertEquals(
(listOf("tagBuild", "distributedPerformanceTests")
+ expectedRunnerParams
+ "-PteamCityBuildId=%teamcity.build.id%"
+ "-PonlyPreviousFailedTestClasses=true"
+ "-PgithubToken=%github.ci.oauth.token%"
+ "-Dscan.tag.RERUN_TESTS"
).joinToString(" "),
performanceTest.getGradleStep("GRADLE_RERUNNER").gradleParams
)
assertEquals(BuildStep.ExecutionMode.RUN_ON_FAILURE, performanceTest.getGradleStep("GRADLE_RERUNNER").executionMode)
}

private
fun BaseGradleBuildType.getGradleStep(stepName: String) = steps.items.find { it.name == stepName }!! as GradleBuildStep
}
Oops, something went wrong.

0 comments on commit 2752984

Please sign in to comment.
You can’t perform that action at this time.