Skip to content
Permalink
Browse files

Merge pull request #152 from abayer/jenkins-41334-mk2

[FIXED JENKINS-41334] Adding parallel stages.
  • Loading branch information...
abayer committed Jul 25, 2017
2 parents e638d4e + 9b046f0 commit 103bec1aeb93f537477c68d2e9222ad97588d2d6
Showing with 967 additions and 136 deletions.
  1. +1 −1 pipeline-model-api/pom.xml
  2. +2 −0 ...del-api/src/main/java/org/jenkinsci/plugins/pipeline/modeldefinition/ast/ModelASTPipelineDef.java
  3. +64 −28 ...ine-model-api/src/main/java/org/jenkinsci/plugins/pipeline/modeldefinition/ast/ModelASTStage.java
  4. +6 −3 ...ne-model-api/src/main/java/org/jenkinsci/plugins/pipeline/modeldefinition/ast/ModelASTStages.java
  5. +1 −1 ...el-api/src/main/java/org/jenkinsci/plugins/pipeline/modeldefinition/validator/ModelValidator.java
  6. +5 −3 pipeline-model-api/src/main/resources/ast-schema.json
  7. +1 −1 pipeline-model-definition/pom.xml
  8. +44 −35 ...line-model-definition/src/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/Utils.groovy
  9. +24 −1 ...efinition/src/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/model/Environment.groovy
  10. +8 −5 ...odel-definition/src/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/model/Stage.groovy
  11. +3 −0 ...efinition/src/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/parser/JSONParser.groovy
  12. +3 −0 ...finition/src/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/parser/ModelParser.groovy
  13. +29 −8 ...rc/main/groovy/org/jenkinsci/plugins/pipeline/modeldefinition/validator/ModelValidatorImpl.groovy
  14. +5 −1 ...-definition/src/main/resources/org/jenkinsci/plugins/pipeline/modeldefinition/Messages.properties
  15. +77 −41 ...inition/src/main/resources/org/jenkinsci/plugins/pipeline/modeldefinition/ModelInterpreter.groovy
  16. +4 −1 ...definition/src/test/java/org/jenkinsci/plugins/pipeline/modeldefinition/AbstractModelDefTest.java
  17. +106 −0 ...el-definition/src/test/java/org/jenkinsci/plugins/pipeline/modeldefinition/BasicModelDefTest.java
  18. +1 −1 ...ion/src/test/java/org/jenkinsci/plugins/pipeline/modeldefinition/BuildConditionResponderTest.java
  19. +24 −0 ...-model-definition/src/test/java/org/jenkinsci/plugins/pipeline/modeldefinition/ValidatorTest.java
  20. +47 −0 pipeline-model-definition/src/test/resources/errors/parallelStagesAgentTools.groovy
  21. +46 −0 pipeline-model-definition/src/test/resources/errors/parallelStagesAndSteps.groovy
  22. +52 −0 pipeline-model-definition/src/test/resources/errors/parallelStagesDeepNesting.groovy
  23. +48 −0 pipeline-model-definition/src/test/resources/json/errors/parallelStagesAgentTools.json
  24. +53 −0 pipeline-model-definition/src/test/resources/json/errors/parallelStagesAndSteps.json
  25. +61 −0 pipeline-model-definition/src/test/resources/json/errors/parallelStagesDeepNesting.json
  26. +40 −0 pipeline-model-definition/src/test/resources/json/nestedParallelStages.json
  27. +43 −0 pipeline-model-definition/src/test/resources/nestedParallelStages.groovy
  28. +97 −0 pipeline-model-definition/src/test/resources/parallelStagesAgentEnvWhen.groovy
  29. +62 −0 pipeline-model-definition/src/test/resources/parallelStagesHaveStatusAndPost.groovy
  30. +1 −1 pipeline-model-extensions/pom.xml
  31. +1 −1 pipeline-model-json-shaded/pom.xml
  32. +1 −1 pipeline-stage-tags-metadata/pom.xml
  33. +1 −1 pom.xml
  34. +6 −2 rfc/JENKINS-41334-parallel-stage-execution.md
@@ -27,7 +27,7 @@
<parent>
<groupId>org.jenkinsci.plugins</groupId>
<artifactId>pipeline-model-parent</artifactId>
<version>1.1.10-SNAPSHOT</version>
<version>1.2-beta-4-SNAPSHOT</version>
</parent>

<groupId>org.jenkinsci.plugins</groupId>
@@ -100,7 +100,9 @@ public String toGroovy() {
result.append(libraries.toGroovy());
}
if (stages != null) {
result.append("stages {\n");
result.append(stages.toGroovy());
result.append("}\n");
}
if (tools != null) {
result.append(tools.toGroovy());
@@ -23,20 +23,27 @@
private ModelASTTools tools;
private ModelASTEnvironment environment;
private Boolean failFast;
private ModelASTStages parallel;

public ModelASTStage(Object sourceLocation) {
super(sourceLocation);
}

@Override
public JSONObject toJSON() {
final JSONArray a = new JSONArray();
for (ModelASTBranch branch : branches) {
a.add(branch.toJSON());
}
JSONObject o = new JSONObject();
o.accumulate("name", name);
o.accumulate("branches", a);

if (branches.isEmpty() && parallel != null) {
o.accumulate("parallel", parallel.toJSON());
} else {
final JSONArray a = new JSONArray();
for (ModelASTBranch branch : branches) {
a.add(branch.toJSON());
}
o.accumulate("branches", a);
}

if (failFast != null) {
o.accumulate("failFast", failFast);
}
@@ -64,10 +71,17 @@ public JSONObject toJSON() {

@Override
public void validate(final ModelValidator validator) {
validator.validateElement(this);
validate(validator, false);
}

public void validate(final ModelValidator validator, boolean isNested) {
validator.validateElement(this, isNested);
for (ModelASTBranch branch : branches) {
branch.validate(validator);
}
if (parallel != null) {
parallel.validate(validator, true);
}
if (agent != null) {
agent.validate(validator);
}
@@ -102,31 +116,37 @@ public String toGroovy() {
if (environment != null) {
result.append(environment.toGroovy());
}
result.append("steps {\n");
if (branches.size() > 1) {
result.append("parallel(");
boolean first = true;
for (ModelASTBranch branch: branches) {
if (first) {
first = false;
} else {
result.append(',');
if (branches.isEmpty() && parallel != null) {
result.append("parallel {\n");
result.append(parallel.toGroovy());
result.append("}\n");
} else {
result.append("steps {\n");
if (branches.size() > 1) {
result.append("parallel(");
boolean first = true;
for (ModelASTBranch branch : branches) {
if (first) {
first = false;
} else {
result.append(',');
}
result.append('\n');
result.append('"' + StringEscapeUtils.escapeJava(branch.getName()) + '"')
.append(": {\n")
.append(branch.toGroovy())
.append("\n}");
}
result.append('\n');
result.append('"' + StringEscapeUtils.escapeJava(branch.getName()) + '"')
.append(": {\n")
.append(branch.toGroovy())
.append("\n}");
}
if (failFast != null && failFast) {
result.append(",\nfailFast: true");
if (failFast != null && failFast) {
result.append(",\nfailFast: true");
}
result.append("\n)\n");
} else if (branches.size() == 1) {
result.append(branches.get(0).toGroovy());
}
result.append("\n)\n");
} else if (branches.size() == 1) {
result.append(branches.get(0).toGroovy());
}

result.append("}\n");
result.append("}\n");
}

if (post != null) {
result.append(post.toGroovy());
@@ -146,6 +166,9 @@ public void removeSourceLocation() {
if (agent != null) {
agent.removeSourceLocation();
}
if (parallel != null) {
parallel.removeSourceLocation();
}
if (when != null) {
when.removeSourceLocation();
}
@@ -224,6 +247,14 @@ public void setFailFast(Boolean f) {
this.failFast = f;
}

public ModelASTStages getParallel() {
return parallel;
}

public void setParallel(ModelASTStages s) {
this.parallel = s;
}

@Override
public String toString() {
return "ModelASTStage{" +
@@ -235,6 +266,7 @@ public String toString() {
", tools=" + tools +
", environment=" + environment +
", failFast=" + failFast +
", parallel=" + parallel +
"}";
}

@@ -268,6 +300,9 @@ public boolean equals(Object o) {
if (getFailFast() != null ? !getFailFast().equals(that.getFailFast()) : that.getFailFast() != null) {
return false;
}
if (getParallel() != null ? !getParallel().equals(that.getParallel()) : that.getParallel() != null) {
return false;
}
if (getTools() != null ? !getTools().equals(that.getTools()) : that.getTools() != null) {
return false;
}
@@ -289,6 +324,7 @@ public int hashCode() {
result = 31 * result + (getTools() != null ? getTools().hashCode() : 0);
result = 31 * result + (getEnvironment() != null ? getEnvironment().hashCode() : 0);
result = 31 * result + (getFailFast() != null ? getFailFast().hashCode() : 0);
result = 31 * result + (getParallel() != null ? getParallel().hashCode() : 0);
return result;
}
}
@@ -28,19 +28,22 @@ public JSONArray toJSON() {

@Override
public void validate(final ModelValidator validator) {
validate(validator, false);
}

public void validate(final ModelValidator validator, boolean isNested) {
validator.validateElement(this);
for (ModelASTStage stage : stages) {
stage.validate(validator);
stage.validate(validator, isNested);
}
}

@Override
public String toGroovy() {
StringBuilder result = new StringBuilder("stages {\n");
StringBuilder result = new StringBuilder();
for (ModelASTStage stage: stages) {
result.append(stage.toGroovy());
}
result.append("}\n");
return result.toString();
}

@@ -89,7 +89,7 @@

boolean validateElement(ModelASTPipelineDef pipelineDef);

boolean validateElement(ModelASTStage stage);
boolean validateElement(ModelASTStage stage, boolean isNested);

boolean validateElement(ModelASTStages stages);

@@ -445,7 +445,7 @@
"additionalProperties": false
},
"stage": {
"description": "A single Pipeline stage, with a name and one or more branches",
"description": "A single Pipeline stage, with a name and either one or more branches or one or more nested stages",
"type": "object",
"properties": {
"name": {
@@ -474,11 +474,13 @@
"items": {
"$ref": "#/definitions/branch"
}
},
"parallel": {
"$ref": "#/definitions/stages"
}
},
"required": [
"name",
"branches"
"name"
],
"additionalProperties": false
},
@@ -27,7 +27,7 @@
<parent>
<groupId>org.jenkinsci.plugins</groupId>
<artifactId>pipeline-model-parent</artifactId>
<version>1.1.10-SNAPSHOT</version>
<version>1.2-beta-4-SNAPSHOT</version>
</parent>

<groupId>org.jenkinsci.plugins</groupId>
@@ -67,14 +67,18 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.when.DeclarativeStageCondi
import org.jenkinsci.plugins.pipeline.modeldefinition.when.DeclarativeStageConditionalDescriptor
import org.jenkinsci.plugins.structs.SymbolLookup
import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable
import org.jenkinsci.plugins.workflow.actions.LabelAction
import org.jenkinsci.plugins.workflow.actions.TagsAction
import org.jenkinsci.plugins.workflow.actions.ThreadNameAction
import org.jenkinsci.plugins.workflow.cps.CpsFlowExecution
import org.jenkinsci.plugins.workflow.cps.CpsScript
import org.jenkinsci.plugins.workflow.cps.CpsThread
import org.jenkinsci.plugins.workflow.flow.FlowExecution
import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner
import org.jenkinsci.plugins.workflow.graph.BlockEndNode
import org.jenkinsci.plugins.workflow.graph.BlockStartNode
import org.jenkinsci.plugins.workflow.graphanalysis.ForkScanner
import org.jenkinsci.plugins.workflow.graphanalysis.LinearBlockHoppingScanner
import org.jenkinsci.plugins.workflow.cps.nodes.StepEndNode
import org.jenkinsci.plugins.workflow.cps.nodes.StepStartNode
import org.jenkinsci.plugins.workflow.graph.FlowNode
import org.jenkinsci.plugins.workflow.job.WorkflowJob
@@ -256,56 +260,50 @@ public class Utils {
}

if (root != null) {
root = populateWhen(root, model)
root = populateEnv(r, root, model)
root = populateFromModel(r, root, model)
}
}

return root
}

static Root populateEnv(@Nonnull WorkflowRun r, @Nonnull Root root, @Nonnull ModelASTPipelineDef model) {
static Root populateFromModel(@Nonnull WorkflowRun r, @Nonnull Root root, @Nonnull ModelASTPipelineDef model) {
root.environment = environmentFromAST(r, model.environment)

List<Stage> stagesWithEnvs = []

root.stages.stages.each { s ->
ModelASTStage astStage = model.stages.stages.find { it.name == s.name }
s.environment = environmentFromAST(r, astStage.environment)
stagesWithEnvs.add(s)
stagesWithEnvs.add(populateSingleStageFromModel(r, model.stages, s))
}

root.stages.stages = stagesWithEnvs

return root
}

/**
* Attaches the {@link StageConditionals} to the appropriate {@link Stage}s, pulling from the AST model.
*
* @param root
* @param model
* @return an updated {@link Root}
*/
static Root populateWhen(@Nonnull Root root, @Nonnull ModelASTPipelineDef model) {
List<Stage> stagesWithWhen = []
private static Stage populateSingleStageFromModel(@Nonnull WorkflowRun r, @Nonnull ModelASTStages allStages,
@Nonnull Stage stage) {
ModelASTStage astStage = allStages.stages.find { it.name == stage.name }
stage.environment = environmentFromAST(r, astStage.environment)

root.stages.stages.each { s ->
ModelASTStage astStage = model.stages.stages.find { it.name == s.name }
if (astStage.when != null) {
List<DeclarativeStageConditional<? extends DeclarativeStageConditional>> processedConditions =
astStage.when.conditions.collect { c ->
stageConditionalFromAST(c)
}
if (astStage.when != null) {
List<DeclarativeStageConditional<? extends DeclarativeStageConditional>> processedConditions =
astStage.when.conditions.collect { c ->
stageConditionalFromAST(c)
}

s.when(new StageConditionals(processedConditions))
}
stagesWithWhen.add(s)
stage.when(new StageConditionals(processedConditions))
}

root.stages.stages = stagesWithWhen
if (stage.parallel != null && !stage.parallel.stages.isEmpty()) {
List<Stage> nestedStages = []
stage.parallel.stages.each { s ->
nestedStages.add(populateSingleStageFromModel(r, astStage.parallel, s))
}
stage.parallel.stages = nestedStages
}

return root
return stage
}

/**
@@ -445,12 +443,12 @@ public class Utils {
}
}

static Predicate<FlowNode> endNodeForStage(final StepStartNode startNode) {
static Predicate<FlowNode> endNodeForStage(final BlockStartNode startNode) {
return new Predicate<FlowNode>() {
@Override
boolean apply(@Nullable FlowNode input) {
return input != null &&
input instanceof StepEndNode &&
input instanceof BlockEndNode &&
input.getStartNode().equals(startNode)
}
}
@@ -460,10 +458,21 @@ public class Utils {
return new Predicate<FlowNode>() {
@Override
boolean apply(@Nullable FlowNode input) {
return input != null &&
input instanceof StepStartNode &&
((StepStartNode) input).descriptor instanceof StageStep.DescriptorImpl &&
(stageName == null || input.displayName?.equals(stageName))
if (input != null) {
if (input instanceof StepStartNode &&
((StepStartNode) input).descriptor instanceof StageStep.DescriptorImpl &&
(stageName == null || input.displayName?.equals(stageName))) {
// This is a true stage.
return true
} else if (input.getAction(LabelAction.class) != null &&
input.getAction(ThreadNameAction.class) != null &&
(stageName == null || input.getAction(ThreadNameAction)?.threadName == stageName)) {
// This is actually a parallel block
return true
}
}

return false
}
}
}
@@ -505,7 +514,7 @@ public class Utils {
CpsThread thread = CpsThread.current()
CpsFlowExecution execution = thread.execution

LinearBlockHoppingScanner scanner = new LinearBlockHoppingScanner();
ForkScanner scanner = new ForkScanner()

return scanner.findFirstMatch(execution.currentHeads, null, isStageWithOptionalName(stageName))
}
Oops, something went wrong.

0 comments on commit 103bec1

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.