Skip to content

Commit

Permalink
use amazon-util version 29
Browse files Browse the repository at this point in the history
  • Loading branch information
@vbedrosova
vbedrosova committed Mar 3, 2017
1 parent 068a780 commit bed57ef
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 134 deletions.
175 changes: 97 additions & 78 deletions ...ine-agent/src/main/java/jetbrains/buildServer/codepipeline/CodePipelineBuildListener.java
View file
Expand Up @@ -16,6 +16,7 @@

package jetbrains.buildServer.codepipeline;

import com.amazonaws.services.codepipeline.AWSCodePipelineClient;
import com.amazonaws.services.codepipeline.model.*;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
Expand All @@ -41,7 +42,6 @@
import static jetbrains.buildServer.codepipeline.CodePipelineConstants.ARTIFACT_INPUT_FOLDER_CONFIG_PARAM;
import static jetbrains.buildServer.codepipeline.CodePipelineConstants.ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM;
import static jetbrains.buildServer.codepipeline.CodePipelineUtil.getJobId;
import static jetbrains.buildServer.util.amazon.AWSCommonParams.createAWSClients;

/**
* @author vbedrosova
Expand Down Expand Up @@ -82,50 +82,59 @@ private void processJobInput(@NotNull AgentRunningBuild build) {
if (myJobInputProcessed) return;
myJobInputProcessed = true;

try {
final Map<String, String> params = build.getSharedConfigParameters();
myJobID = getJobId(params);
if (myJobID == null) {
LOG.debug(msgForBuild("No AWS CodePipeline job found for the build", build));
return;
}

final JobData jobData = getJobData(params);

final PipelineContext pipelineContext = jobData.getPipelineContext();
build.getBuildLogger().message(
"This build is a part of an AWS CodePipeline pipeline: " + pipelineContext.getPipelineName() +
"\nLink: https://console.aws.amazon.com/codepipeline/home?region=" + params.get(AWSCommonParams.REGION_NAME_PARAM) + "#/view/" + pipelineContext.getPipelineName() +
"\nStage: " + pipelineContext.getStage().getName() +
"\nAction: " + pipelineContext.getAction().getName() +
"\nJob ID: " + myJobID);

final List<Artifact> inputArtifacts = jobData.getInputArtifacts();
if (inputArtifacts.isEmpty()) {
LOG.debug(msgForBuild("No input artifacts provided for the job with ID: " + myJobID, build));
} else {

final File inputFolder = new File(params.get(ARTIFACT_INPUT_FOLDER_CONFIG_PARAM));
FileUtil.createDir(inputFolder);

for (Artifact artifact : inputArtifacts) {
final S3ArtifactLocation s3Location = artifact.getLocation().getS3Location();
final File destinationFile = getInputArtifactFile(inputFolder, s3Location);

build.getBuildLogger().message("Downloading job input artifact " + s3Location.getObjectKey() + " to " + destinationFile.getAbsolutePath());
getArtifactS3Client(jobData.getArtifactCredentials(), params)
.getObject(new GetObjectRequest(s3Location.getBucketName(), s3Location.getObjectKey()), destinationFile);
final Map<String, String> params = build.getSharedConfigParameters();
myJobID = getJobId(params);
if (myJobID == null) {
LOG.debug(msgForBuild("No AWS CodePipeline job found for the build", build));
return;
}

// for backward compatibility, TW-47902
makeArtifactCopy(inputFolder, destinationFile, s3Location.getObjectKey());
}
if (!jobData.getOutputArtifacts().isEmpty()) {
FileUtil.createDir(new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM)));
AWSCommonParams.withAWSClients(params, new AWSCommonParams.WithAWSClients<Void, RuntimeException>() {
@Nullable
@Override
public Void run(@NotNull AWSClients clients) throws RuntimeException {
AWSCodePipelineClient codePipelineClient = null;
try {
codePipelineClient = clients.createCodePipeLineClient();
final JobData jobData = getJobData(codePipelineClient, params);

final PipelineContext pipelineContext = jobData.getPipelineContext();
build.getBuildLogger().message(
"This build is a part of an AWS CodePipeline pipeline: " + pipelineContext.getPipelineName() +
"\nLink: https://console.aws.amazon.com/codepipeline/home?region=" + params.get(AWSCommonParams.REGION_NAME_PARAM) + "#/view/" + pipelineContext.getPipelineName() +
"\nStage: " + pipelineContext.getStage().getName() +
"\nAction: " + pipelineContext.getAction().getName() +
"\nJob ID: " + myJobID);

final List<Artifact> inputArtifacts = jobData.getInputArtifacts();
if (inputArtifacts.isEmpty()) {
LOG.debug(msgForBuild("No input artifacts provided for the job with ID: " + myJobID, build));
} else {

final File inputFolder = new File(params.get(ARTIFACT_INPUT_FOLDER_CONFIG_PARAM));
FileUtil.createDir(inputFolder);

for (Artifact artifact : inputArtifacts) {
final S3ArtifactLocation s3Location = artifact.getLocation().getS3Location();
final File destinationFile = getInputArtifactFile(inputFolder, s3Location);

build.getBuildLogger().message("Downloading job input artifact " + s3Location.getObjectKey() + " to " + destinationFile.getAbsolutePath());
getArtifactS3Client(jobData.getArtifactCredentials(), params)
.getObject(new GetObjectRequest(s3Location.getBucketName(), s3Location.getObjectKey()), destinationFile);

// for backward compatibility, TW-47902
makeArtifactCopy(inputFolder, destinationFile, s3Location.getObjectKey());
}
if (!jobData.getOutputArtifacts().isEmpty()) {
FileUtil.createDir(new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM)));
}
}
} catch (Throwable e) {
failOnException(codePipelineClient, build, e);
}
return null;
}
} catch (Throwable e) {
failOnException(build, e);
}
});
}

@NotNull
Expand All @@ -142,37 +151,45 @@ private void makeArtifactCopy(@NotNull File inputFolder, @NotNull File artifactF
private void processJobOutput(@NotNull AgentRunningBuild build, @NotNull BuildFinishedStatus buildStatus) {
if (myJobID == null) return;

try {
if (build.isBuildFailingOnServer()) {
publishJobFailure(build, "Build failed");
} else if (BuildFinishedStatus.INTERRUPTED == buildStatus) {
publishJobFailure(build, "Build interrupted");
} else {
final Map<String, String> params = build.getSharedConfigParameters();
final JobData jobData = getJobData(params);

final List<Artifact> outputArtifacts = jobData.getOutputArtifacts();
if (outputArtifacts.isEmpty()) {
LOG.debug(msgForBuild("No output artifacts expected for the job with ID: " + myJobID, build));
} else {
final File artifactOutputFolder = new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM));
for (Artifact artifact : outputArtifacts) {
final File buildArtifact = getBuildArtifact(artifact, jobData.getPipelineContext().getPipelineName(), artifactOutputFolder, build);
final S3ArtifactLocation s3Location = artifact.getLocation().getS3Location();

build.getBuildLogger().message("Uploading job output artifact " + s3Location.getObjectKey() + " from " + buildArtifact.getAbsolutePath());
getArtifactS3Client(jobData.getArtifactCredentials(), params)
.putObject(new PutObjectRequest(s3Location.getBucketName(), s3Location.getObjectKey(), buildArtifact)
.withSSEAwsKeyManagementParams(getSSEAwsKeyManagementParams(jobData.getEncryptionKey())));
AWSCommonParams.withAWSClients(build.getSharedConfigParameters(), new AWSCommonParams.WithAWSClients<Void, RuntimeException>() {
@Nullable
@Override
public Void run(@NotNull AWSClients clients) throws RuntimeException {
AWSCodePipelineClient codePipelineClient = null;
try {
codePipelineClient = clients.createCodePipeLineClient();
if (build.isBuildFailingOnServer()) {
publishJobFailure(codePipelineClient, build, "Build failed");
} else if (BuildFinishedStatus.INTERRUPTED == buildStatus) {
publishJobFailure(codePipelineClient, build, "Build interrupted");
} else {
final Map<String, String> params = build.getSharedConfigParameters();
final JobData jobData = getJobData(codePipelineClient, params);

final List<Artifact> outputArtifacts = jobData.getOutputArtifacts();
if (outputArtifacts.isEmpty()) {
LOG.debug(msgForBuild("No output artifacts expected for the job with ID: " + myJobID, build));
} else {
final File artifactOutputFolder = new File(params.get(ARTIFACT_OUTPUT_FOLDER_CONFIG_PARAM));
for (Artifact artifact : outputArtifacts) {
final File buildArtifact = getBuildArtifact(artifact, jobData.getPipelineContext().getPipelineName(), artifactOutputFolder, build);
final S3ArtifactLocation s3Location = artifact.getLocation().getS3Location();

build.getBuildLogger().message("Uploading job output artifact " + s3Location.getObjectKey() + " from " + buildArtifact.getAbsolutePath());
getArtifactS3Client(jobData.getArtifactCredentials(), params)
.putObject(new PutObjectRequest(s3Location.getBucketName(), s3Location.getObjectKey(), buildArtifact)
.withSSEAwsKeyManagementParams(getSSEAwsKeyManagementParams(jobData.getEncryptionKey())));
}

publishJobSuccess(codePipelineClient, build);
}
}

publishJobSuccess(build);
} catch (Throwable e) {
failOnException(codePipelineClient, build, e);
}
return null;
}

} catch (Throwable e) {
failOnException(build, e);
}
});
}

@NotNull
Expand All @@ -181,17 +198,17 @@ private static SSEAwsKeyManagementParams getSSEAwsKeyManagementParams(@Nullable
? new SSEAwsKeyManagementParams() : new SSEAwsKeyManagementParams(encryptionKey.getId());
}

private void publishJobSuccess(@NotNull AgentRunningBuild build) {
createAWSClients(build.getSharedConfigParameters()).createCodePipeLineClient().putJobSuccessResult(
private void publishJobSuccess(@NotNull AWSCodePipelineClient codePipelineClient, @NotNull AgentRunningBuild build) {
codePipelineClient.putJobSuccessResult(
new PutJobSuccessResultRequest().withJobId(myJobID).withExecutionDetails(
new ExecutionDetails().withExternalExecutionId(String.valueOf(build.getBuildId())).withSummary("Build successfully finished")
)
);
}

private void publishJobFailure(@NotNull AgentRunningBuild build, @NotNull String message) {
private void publishJobFailure(@NotNull AWSCodePipelineClient codePipelineClient, @NotNull AgentRunningBuild build, @NotNull String message) {
try {
createAWSClients(build.getSharedConfigParameters()).createCodePipeLineClient().putJobFailureResult(
codePipelineClient.putJobFailureResult(
new PutJobFailureResultRequest().withJobId(myJobID).withFailureDetails(
new FailureDetails()
.withExternalExecutionId(String.valueOf(build.getBuildId()))
Expand All @@ -207,7 +224,7 @@ private void publishJobFailure(@NotNull AgentRunningBuild build, @NotNull String
}
}

private void failOnException(@NotNull AgentRunningBuild build, @NotNull Throwable cause) {
private void failOnException(@Nullable AWSCodePipelineClient codePipelineClient, @NotNull AgentRunningBuild build, @NotNull Throwable cause) {
final AWSException e = new AWSException(cause);

LOG.error(msgForBuild(e.getMessage(), build));
Expand All @@ -223,7 +240,9 @@ private void failOnException(@NotNull AgentRunningBuild build, @NotNull Throwabl

build.getBuildLogger().logBuildProblem(createBuildProblem(build, e));

publishJobFailure(build, e.getMessage());
if (codePipelineClient != null) {
publishJobFailure(codePipelineClient, build, e.getMessage());
}
}

@NotNull
Expand Down Expand Up @@ -253,8 +272,8 @@ private String msgForBuild(@NotNull String msg, @NotNull AgentRunningBuild build
}

@NotNull
private JobData getJobData(@NotNull Map<String, String> params) {
return createAWSClients(params).createCodePipeLineClient().getJobDetails(new GetJobDetailsRequest().withJobId(myJobID)).getJobDetails().getData();
private JobData getJobData(@NotNull AWSCodePipelineClient codePipelineClient, @NotNull Map<String, String> params) {
return codePipelineClient.getJobDetails(new GetJobDetailsRequest().withJobId(myJobID)).getJobDetails().getData();
}

@NotNull
Expand Down
112 changes: 57 additions & 55 deletions ...jetbrains/buildServer/buildTriggers/codepipeline/CodePipelineAsyncPolledBuildTrigger.java
View file
Expand Up @@ -59,65 +59,67 @@ public CodePipelineAsyncPolledBuildTrigger(@NotNull BuildCustomizerFactory build
@Nullable
@Override
public String triggerBuild(@Nullable String previousValue, @NotNull PolledTriggerContext context) throws BuildTriggerException {
final Map<String, String> properties = validateParams(context.getTriggerDescriptor().getProperties());
try {
final Map<String, String> properties = context.getTriggerDescriptor().getProperties();
final AWSCodePipelineClient codePipelineClient =
AWSCommonParams.createAWSClients(validateParams(properties)).createCodePipeLineClient();

final PollForJobsRequest request = new PollForJobsRequest()
.withActionTypeId(
new ActionTypeId()
.withCategory(ActionCategory.Build)
.withOwner(ActionOwner.Custom)
.withProvider(TEAMCITY_ACTION_PROVIDER)
.withVersion(getActionTypeVersion(codePipelineClient)))
.withQueryParam(CollectionsUtil.<String>asMap(
ACTION_TOKEN_CONFIG_PROPERTY, CodePipelineUtil.getActionToken(properties)))
.withMaxBatchSize(1);

final List<Job> jobs = codePipelineClient.pollForJobs(request).getJobs();

if (jobs.size() > 0) {
if (jobs.size() > 1) {
LOG.warn(msgForBt("Received " + jobs.size() + ", but only one was expected. Will process only the first job", context.getBuildType()));
}

final Job job = jobs.get(0);
LOG.info(msgForBt("Received job request with ID: " + job.getId() + " and nonce: " + job.getNonce(), context.getBuildType()));

try {
final AcknowledgeJobRequest acknowledgeJobRequest = new AcknowledgeJobRequest()
.withJobId(job.getId())
.withNonce(job.getNonce());

final String jobStatus = codePipelineClient.acknowledgeJob(acknowledgeJobRequest).getStatus();
if (jobStatus.equals(JobStatus.InProgress.name())) {

final BuildCustomizer buildCustomizer = myBuildCustomizerFactory.createBuildCustomizer(context.getBuildType(), null);
buildCustomizer.setParameters(getCustomBuildParameters(job, context));

final BuildPromotion promotion = buildCustomizer.createPromotion();
promotion.addToQueue(TRIGGER_DISPLAY_NAME + " job with ID: " + job.getId());

LOG.info(msgForBt("Acknowledged job with ID: " + job.getId()+ " and nonce: " + job.getNonce() + ", created build promotion " + promotion.getId(), context.getBuildType()));
AWSCommonParams.withAWSClients(properties, clients -> {
final AWSCodePipelineClient codePipelineClient = clients.createCodePipeLineClient();

final PollForJobsRequest request = new PollForJobsRequest()
.withActionTypeId(
new ActionTypeId()
.withCategory(ActionCategory.Build)
.withOwner(ActionOwner.Custom)
.withProvider(TEAMCITY_ACTION_PROVIDER)
.withVersion(getActionTypeVersion(codePipelineClient)))
.withQueryParam(CollectionsUtil.asMap(
ACTION_TOKEN_CONFIG_PROPERTY, CodePipelineUtil.getActionToken(properties)))
.withMaxBatchSize(1);

final List<Job> jobs = codePipelineClient.pollForJobs(request).getJobs();

if (jobs.size() > 0) {
if (jobs.size() > 1) {
LOG.warn(msgForBt("Received " + jobs.size() + ", but only one was expected. Will process only the first job", context.getBuildType()));
}

} else {
LOG.warn(msgForBt("Job ignored with ID: " + job.getId()+ " and nonce: " + job.getNonce() + " because job status is " + jobStatus, context.getBuildType()));
final Job job = jobs.get(0);
LOG.info(msgForBt("Received job request with ID: " + job.getId() + " and nonce: " + job.getNonce(), context.getBuildType()));

try {
final AcknowledgeJobRequest acknowledgeJobRequest = new AcknowledgeJobRequest()
.withJobId(job.getId())
.withNonce(job.getNonce());

final String jobStatus = codePipelineClient.acknowledgeJob(acknowledgeJobRequest).getStatus();
if (jobStatus.equals(JobStatus.InProgress.name())) {

final BuildCustomizer buildCustomizer = myBuildCustomizerFactory.createBuildCustomizer(context.getBuildType(), null);
buildCustomizer.setParameters(getCustomBuildParameters(job, context));

final BuildPromotion promotion = buildCustomizer.createPromotion();
promotion.addToQueue(TRIGGER_DISPLAY_NAME + " job with ID: " + job.getId());

LOG.info(msgForBt("Acknowledged job with ID: " + job.getId()+ " and nonce: " + job.getNonce() + ", created build promotion " + promotion.getId(), context.getBuildType()));

} else {
LOG.warn(msgForBt("Job ignored with ID: " + job.getId()+ " and nonce: " + job.getNonce() + " because job status is " + jobStatus, context.getBuildType()));
}
} catch (Throwable e) {
final BuildTriggerException buildTriggerException = processThrowable(e);
codePipelineClient.putJobFailureResult(
new PutJobFailureResultRequest().withJobId(job.getId()).withFailureDetails(
new FailureDetails()
.withType(FailureType.JobFailed)
.withMessage(buildTriggerException.getMessage())
)
);
throw buildTriggerException;
}
} catch (Throwable e) {
final BuildTriggerException buildTriggerException = processThrowable(e);
codePipelineClient.putJobFailureResult(
new PutJobFailureResultRequest().withJobId(job.getId()).withFailureDetails(
new FailureDetails()
.withType(FailureType.JobFailed)
.withMessage(buildTriggerException.getMessage())
)
);
throw buildTriggerException;
} else {
LOG.debug(msgForBt("No jobs found", context.getBuildType()));
}
} else {
LOG.debug(msgForBt("No jobs found", context.getBuildType()));
}
return null;
});
} catch (Throwable e) {
throw processThrowable(e);
}
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
View file
Expand Up @@ -34,7 +34,7 @@ ext.teamcityJavaHome = System.properties['java.home']

ext.javaVersion = hasProperty('plugin.java.version') ? property('plugin.java.version') : '1.8'

ext.amazonUtilVersion = hasProperty('amazon.util.version') ? property('amazon.util.version') : '22'
ext.amazonUtilVersion = hasProperty('amazon.util.version') ? property('amazon.util.version') : '29'


apply plugin: 'idea'
Expand Down

0 comments on commit bed57ef

Please sign in to comment.