Skip to content

Commit

Permalink
[TASK] Run tests on new bamboo infrastructure
Browse files Browse the repository at this point in the history 
A new bamboo agent infrastructure has been deployed that
significantly changes how tests are executed: The agent
docker containers are now "stupid" and no longer bundle
specific php versions or daemons. Instead, they can run
own containers to start needed daemons for specific jobs
and execute needed php commands in ad-hoc containers that
provide the required php version.
Daemons needed for single jobs are defined in a
docker-compose.yml file provided by core itself.
This docker-compose.yml file can not be used directly for
local test execution since it has to fiddle quite a bit
with docker volume mounts, networks and executing users
that is specific to the bamboo environment.
However, another yml file can be added later to ease local
test execution in a similar way.
The patch rewrites the bamboo plan pre-merge and nightly
specs of core master to use the new infrastructure and brings
a couple of minor changes to tests that rely on a running
memcached or redis to retrieve the daemon host from an
environment variable.
There will be separated patche for older core versions.

Change-Id: I65777eeee6e28fca5b3d3d979498293cc91a77af
Resolves: #85563
Resolves: #36934
Releases: master
Reviewed-on: https://review.typo3.org/57522
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
  • Loading branch information
@lolli42
lolli42 committed Jul 15, 2018
1 parent 0f811d4 commit 1819365
Show file tree
Hide file tree
Showing 12 changed files with 878 additions and 379 deletions.
814 changes: 563 additions & 251 deletions Build/bamboo/src/main/java/core/AbstractCoreSpec.java
View file

Large diffs are not rendered by default.

72 changes: 36 additions & 36 deletions Build/bamboo/src/main/java/core/NightlySpec.java
View file
Expand Up @@ -66,49 +66,37 @@ Project project() {
* Returns full Plan definition
*/
Plan createPlan() {
// PREPARATION stage
ArrayList<Job> jobsPreparationStage = new ArrayList<Job>();

jobsPreparationStage.add(this.getJobComposerValidate());

Stage stagePreparation = new Stage("Preparation")
.jobs(jobsPreparationStage.toArray(new Job[jobsPreparationStage.size()]));


// MAIN stage
ArrayList<Job> jobsMainStage = new ArrayList<Job>();

jobsMainStage.add(this.getJobAcceptanceTestInstallMysql(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite(this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.add(this.getJobCglCheckFullCore());

jobsMainStage.add(this.getJobIntegrationAnnotations());

jobsMainStage.add(this.getJobIntegrationVarious());
jobsMainStage.add(this.getJobComposerValidate("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallMysql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, "PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobCglCheckFullCore("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobIntegrationAnnotations("PHP72"));

jobsMainStage.add(this.getJobUnitJavaScript());
jobsMainStage.add(this.getJobIntegrationVarious("PHP72"));

jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, "PHP72"));

jobsMainStage.add(this.getJobLintScssTs());
jobsMainStage.add(this.getJobUnitJavaScript("PHP72"));

jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobLintPhp("PHP72"));

jobsMainStage.add(this.getJobUnitDeprecatedPhp(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobLintScssTs("PHP72"));

jobsMainStage.addAll(this.getJobUnitPhpRandom(this.numberOfUnitRandomOrderJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobUnitPhp("PHP72"));
jobsMainStage.add(this.getJobUnitDeprecatedPhp("PHP72"));
jobsMainStage.addAll(this.getJobUnitPhpRandom(this.numberOfUnitRandomOrderJobs, "PHP72"));

Stage stageMainStage = new Stage("Main stage")
.jobs(jobsMainStage.toArray(new Job[jobsMainStage.size()]));
Expand All @@ -119,7 +107,6 @@ Plan createPlan() {
.description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
.pluginConfigurations(this.getDefaultPlanPluginConfiguration())
.stages(
stagePreparation,
stageMainStage
)
.linkedRepositories("git.typo3.org Core")
Expand All @@ -143,26 +130,39 @@ Plan createPlan() {

/**
* Job checking CGL of all core php files
*
* @param String requirementIdentifier
*/
protected Job getJobCglCheckFullCore() {
protected Job getJobCglCheckFullCore(String requirementIdentifier) {
return new Job("Integration CGL", new BambooKey("CGLCHECK"))
.description("Check coding guidelines of full core")
.pluginConfigurations(this.getDefaultJobPluginConfiguration())
.tasks(
this.getTaskGitCloneRepository(),
this.getTaskGitCherryPick(),
this.getTaskComposerInstall(),
this.getTaskComposerInstall(requirementIdentifier),
new ScriptTask()
.description("Execute cgl check")
.interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
.inlineBody(
this.getScriptTaskBashInlineBody() +
"php -n -c /etc/php/cli-no-xdebug/php.ini bin/php-cs-fixer fix -v --dry-run --path-mode intersection --config=Build/.php_cs typo3/\n" +
"exit $?\n"
"function phpCsFixer() {\n" +
" docker run \\\n" +
" -u ${HOST_UID} \\\n" +
" -v /etc/passwd:/etc/passwd \\\n" +
" -v ${BAMBOO_COMPOSE_PROJECT_NAME}_bamboo-data:/srv/bamboo/xml-data/build-dir/ \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
" --rm \\\n" +
" typo3gmbh/" + requirementIdentifier.toLowerCase() + ":latest \\\n" +
" bin/bash -c \"cd ${PWD}; php -n -c /etc/php/cli-no-xdebug/php.ini bin/php-cs-fixer $*\"\n" +
"}\n" +
"\n" +
"phpCsFixer fix -v --dry-run --path-mode intersection --config=Build/.php_cs typo3/\n" +
"exit $?"
)
)
.requirements(
this.getRequirementPhpVersion72()
this.getRequirementDocker10()
)
.cleanWorkingDirectory(true);
}
Expand Down
75 changes: 44 additions & 31 deletions Build/bamboo/src/main/java/core/PreMergeSpec.java
View file
Expand Up @@ -76,48 +76,44 @@ Project project() {
Plan createPlan() {
// PREPARATION stage
ArrayList<Job> jobsPreparationStage = new ArrayList<Job>();

jobsPreparationStage.add(this.getJobBuildLabels());

jobsPreparationStage.add(this.getJobCglCheckGitCommit());

jobsPreparationStage.add(this.getJobComposerValidate());

Stage stagePreparation = new Stage("Preparation")
.jobs(jobsPreparationStage.toArray(new Job[jobsPreparationStage.size()]));

// EARLY stage
ArrayList<Job> jobsEarlyStage = new ArrayList<Job>();
jobsEarlyStage.add(this.getJobCglCheckGitCommit("PHP72"));
jobsEarlyStage.add(this.getJobComposerValidate("PHP72"));
Stage stageEarly = new Stage("Early")
.jobs(jobsEarlyStage.toArray(new Job[jobsEarlyStage.size()]));

// MAIN stage
ArrayList<Job> jobsMainStage = new ArrayList<Job>();

jobsMainStage.add(this.getJobAcceptanceTestInstallMysql(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql(this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite(this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallMysql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite("PHP72"));

jobsMainStage.add(this.getJobIntegrationAnnotations());
jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, "PHP72"));

jobsMainStage.add(this.getJobIntegrationVarious());
jobsMainStage.add(this.getJobIntegrationAnnotations("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobIntegrationVarious("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, "PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobUnitJavaScript("PHP72"));

jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobLintPhp("PHP72"));

jobsMainStage.add(this.getJobUnitJavaScript());
jobsMainStage.add(this.getJobLintScssTs("PHP72"));

jobsMainStage.add(this.getJobLintPhp(this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.add(this.getJobLintScssTs());

jobsMainStage.add(this.getJobUnitPhp(this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.add(this.getJobUnitDeprecatedPhp(this.getRequirementPhpVersion72(), "PHP72"));

jobsMainStage.addAll(this.getJobUnitPhpRandom(this.numberOfUnitRandomOrderJobs, this.getRequirementPhpVersion72(), "PHP72"));
jobsMainStage.add(this.getJobUnitPhp("PHP72"));
jobsMainStage.add(this.getJobUnitDeprecatedPhp("PHP72"));
jobsMainStage.addAll(this.getJobUnitPhpRandom(this.numberOfUnitRandomOrderJobs, "PHP72"));

Stage stageMainStage = new Stage("Main stage")
.jobs(jobsMainStage.toArray(new Job[jobsMainStage.size()]));
Expand All @@ -128,6 +124,7 @@ Plan createPlan() {
.pluginConfigurations(this.getDefaultPlanPluginConfiguration())
.stages(
stagePreparation,
stageEarly,
stageMainStage
)
.linkedRepositories("git.typo3.org Core")
Expand Down Expand Up @@ -188,30 +185,46 @@ protected Job getJobBuildLabels() {
.interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
.inlineBody("echo \"I'm just here for the labels!\"")
)
.requirements(
this.getRequirementDocker10()
)
.cleanWorkingDirectory(true);
}

/**
* Job checking CGL of last git commit
*
* @param String requirementIdentifier
*/
protected Job getJobCglCheckGitCommit() {
protected Job getJobCglCheckGitCommit(String requirementIdentifier) {
return new Job("Integration CGL", new BambooKey("CGLCHECK"))
.description("Check coding guidelines by executing Build/Scripts/cglFixMyCommit.sh script")
.pluginConfigurations(this.getDefaultJobPluginConfiguration())
.tasks(
this.getTaskGitCloneRepository(),
this.getTaskGitCherryPick(),
this.getTaskComposerInstall(),
this.getTaskComposerInstall(requirementIdentifier),
new ScriptTask()
.description("Execute cgl check script")
.interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
.inlineBody(
this.getScriptTaskBashInlineBody() +
"./Build/Scripts/cglFixMyCommit.sh dryrun\n"
"function cglFixMyCommit() {\n" +
" docker run \\\n" +
" -u ${HOST_UID} \\\n" +
" -v /etc/passwd:/etc/passwd \\\n" +
" -v ${BAMBOO_COMPOSE_PROJECT_NAME}_bamboo-data:/srv/bamboo/xml-data/build-dir/ \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
" --rm \\\n" +
" typo3gmbh/" + requirementIdentifier.toLowerCase() + ":latest \\\n" +
" bin/bash -c \"cd ${PWD}; ./Build/Scripts/cglFixMyCommit.sh $*\"\n" +
"}\n" +
"\n" +
"cglFixMyCommit dryrun\n"
)
)
.requirements(
this.getRequirementPhpVersion72()
this.getRequirementDocker10()
)
.cleanWorkingDirectory(true);
}
Expand Down

0 comments on commit 1819365

Please sign in to comment.