Permalink
Browse files

[JENKINS-24786] Environment variables are not expanded in git publisher

Fix: Git publisher fails if remote repository configuration contains
environment variables
  • Loading branch information...
dfigus committed Dec 22, 2014
1 parent 0c09974 commit f841caf3c2483532aad9feb20d2f3fac9722a878
@@ -232,6 +232,10 @@ public boolean perform(AbstractBuild<?, ?> build,

if (mergeOptions.doMerge() && buildResult.isBetterOrEqualTo(Result.SUCCESS)) {
RemoteConfig remote = mergeOptions.getMergeRemote();

// expand environment variables in remote repository
remote = gitSCM.getParamExpandedRepo(environment, remote);

listener.getLogger().println("Pushing HEAD to branch " + mergeTarget + " of " + remote.getName() + " repository");

remoteURI = remote.getURIs().get(0);
@@ -266,11 +270,15 @@ public boolean perform(AbstractBuild<?, ?> build,
final String targetRepo = environment.expand(t.getTargetRepoName());

try {
RemoteConfig remote = gitSCM.getRepositoryByName(targetRepo);
// Lookup repository with unexpanded name as GitSCM stores them unexpanded
RemoteConfig remote = gitSCM.getRepositoryByName(t.getTargetRepoName());

if (remote == null)
throw new AbortException("No repository found for target repo name " + targetRepo);


// expand environment variables in remote repository
remote = gitSCM.getParamExpandedRepo(environment, remote);

boolean tagExists = git.tagExists(tagName.replace(' ', '_'));
if (t.isCreateTag() || t.isUpdateTag()) {
if (tagExists && !t.isUpdateTag()) {
@@ -315,11 +323,15 @@ else if (!tagExists) {
final String targetRepo = environment.expand(b.getTargetRepoName());

try {
RemoteConfig remote = gitSCM.getRepositoryByName(targetRepo);
// Lookup repository with unexpanded name as GitSCM stores them unexpanded
RemoteConfig remote = gitSCM.getRepositoryByName(b.getTargetRepoName());

if (remote == null)
throw new AbortException("No repository found for target repo name " + targetRepo);


// expand environment variables in remote repository
remote = gitSCM.getParamExpandedRepo(environment, remote);

listener.getLogger().println("Pushing HEAD to branch " + branchName + " at repo "
+ targetRepo);
remoteURI = remote.getURIs().get(0);
@@ -348,13 +360,17 @@ else if (!tagExists) {
final boolean noteReplace = b.getnoteReplace();

try {
RemoteConfig remote = gitSCM.getRepositoryByName(targetRepo);
// Lookup repository with unexpanded name as GitSCM stores them unexpanded
RemoteConfig remote = gitSCM.getRepositoryByName(b.getTargetRepoName());

if (remote == null) {
listener.getLogger().println("No repository found for target repo name " + targetRepo);
return false;
}


// expand environment variables in remote repository
remote = gitSCM.getParamExpandedRepo(environment, remote);

listener.getLogger().println("Adding note to namespace \""+noteNamespace +"\":\n" + noteMsg + "\n******" );

if ( noteReplace )
@@ -392,15 +392,24 @@ public String getParamLocalBranch(Run<?, ?> build, TaskListener listener) throws
EnvVars env = build.getEnvironment(listener);

for (RemoteConfig oldRepo : Util.fixNull(remoteRepositories)) {
expandedRepos.add(
newRemoteConfig(
getParameterString(oldRepo.getName(), env),
getParameterString(oldRepo.getURIs().get(0).toPrivateString(), env),
getRefSpecs(oldRepo, env).toArray(new RefSpec[0])));
expandedRepos.add(getParamExpandedRepo(env, oldRepo));
}

return expandedRepos;
}

/**
* Expand Parameters in the supplied remote repository with the parameter values provided in the given environment variables }
* @param env Environment variables with parameter values
* @param remoteRepository Remote repository with parameters
* @return remote repository with expanded parameters
*/
public RemoteConfig getParamExpandedRepo(EnvVars env, RemoteConfig remoteRepository){
return newRemoteConfig(
getParameterString(remoteRepository.getName(), env),
getParameterString(remoteRepository.getURIs().get(0).toPrivateString(), env),
getRefSpecs(remoteRepository, env).toArray(new RefSpec[0]));
}

public RemoteConfig getRepositoryByName(String repoName) {
for (RemoteConfig r : getRepositories()) {
@@ -39,6 +39,7 @@
import hudson.plugins.git.extensions.impl.PreBuildMerge;
import hudson.scm.NullSCM;
import hudson.tasks.BuildStepDescriptor;

import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.jvnet.hudson.test.Bug;
@@ -137,6 +138,54 @@ public void testMergeAndPush() throws Exception {

}

@Issue("JENKINS-24786")
public void testPushEnvVarsInRemoteConfig() throws Exception{
FreeStyleProject project = setupSimpleProject("master");

// create second (bare) test repository as target
TestGitRepo testTargetRepo = new TestGitRepo("target", this, listener);
testTargetRepo.git.init_().workspace(testTargetRepo.gitDir.getAbsolutePath()).bare(true).execute();
testTargetRepo.commit("lostTargetFile", johnDoe, "Initial Target Commit");

// add second test repository as remote repository with environment variables
List<UserRemoteConfig> remoteRepositories = createRemoteRepositories();
remoteRepositories.add(new UserRemoteConfig("$TARGET_URL", "$TARGET_NAME", "+refs/heads/$TARGET_BRANCH:refs/remotes/$TARGET_NAME/$TARGET_BRANCH", null));

GitSCM scm = new GitSCM(
remoteRepositories,
Collections.singletonList(new BranchSpec("origin/master")),
false, Collections.<SubmoduleConfig>emptyList(),
null, null,
Collections.<GitSCMExtension>emptyList());
project.setScm(scm);

// add parameters for remote repository configuration
project.addProperty(new ParametersDefinitionProperty(
new StringParameterDefinition("TARGET_URL", testTargetRepo.gitDir.getAbsolutePath()),
new StringParameterDefinition("TARGET_NAME", "target"),
new StringParameterDefinition("TARGET_BRANCH", "master")));

String tag_name = "test-tag";
String note_content = "Test Note";

project.getPublishersList().add(new GitPublisher(
Collections.singletonList(new TagToPush("$TARGET_NAME", tag_name, "", false, false)),
Collections.singletonList(new BranchToPush("$TARGET_NAME", "$TARGET_BRANCH")),
Collections.singletonList(new NoteToPush("$TARGET_NAME", note_content, Constants.R_NOTES_COMMITS, false)),
true, false, true));

commit("commitFile", johnDoe, "Initial Commit");
testRepo.git.tag(tag_name, "Comment");
ObjectId expectedCommit = testRepo.git.revParse("master");

build(project, Result.SUCCESS, "commitFile");

// check if everything reached target repository
assertEquals(expectedCommit, testTargetRepo.git.revParse("master"));
assertTrue(existsTagInRepo(testTargetRepo, tag_name));

}

@Issue("JENKINS-24082")
public void testForcePush() throws Exception {
FreeStyleProject project = setupSimpleProject("master");
@@ -313,10 +362,14 @@ private void checkEnvVar(FreeStyleProject project, String envName, String envVal
}

private boolean existsTag(String tag) throws InterruptedException {
Set<String> tags = git.getTagNames("*");
return existsTagInRepo(testRepo, tag);
}

private boolean existsTagInRepo(TestGitRepo gitRepo, String tag) throws InterruptedException {
Set<String> tags = gitRepo.git.getTagNames("*");
return tags.contains(tag);
}

private boolean containsTagMessage(String tag, String str) throws InterruptedException {
String msg = git.getTagMessage(tag);
return msg.contains(str);

9 comments on commit f841caf

@tmertens

This comment has been minimized.

Copy link

tmertens replied Jan 2, 2015

@MarkEWaite Any chance we can get a new release soon with this patch included? I'm trying to set up parameterized builds for auto-deploy to heroku and this is a blocker on the publishing side.

Keep up the great work!

@MarkEWaite

This comment has been minimized.

Copy link

MarkEWaite replied Jan 2, 2015

A version of the plugin containing this change will likely release in a week or less.

Have you confirmed (with a pre-release build) that the change meets your needs? If not, would you be willing to test drive a pre-release if one were available within the next few days?

@tmertens

This comment has been minimized.

Copy link

tmertens replied Jan 2, 2015

I have not tried building the plugin myself, no. I just recently installed jenkins on AWS and am trying it out for auto-deploy. I tried updating from 2.2.10 to 2.3.2 to see if it resolved the issue. It didn't, so I glanced at the code and saw this recent patch was committed after the 2.3.2 release and appears to resolve the issue. I'd be happy to give a pre-release version a try if one is made available.

@MarkEWaite

This comment has been minimized.

Copy link

MarkEWaite replied Jan 8, 2015

The git-plugin 2.3.3 and later includes this change

@tmertens

This comment has been minimized.

Copy link

tmertens replied Jan 8, 2015

Thanks. I actually built it a couple days ago and installed it, just upgraded to the official release. This resolves the issue I was seeing but now I'm hitting the timeout identified here: https://issues.jenkins-ci.org/browse/JENKINS-23476

Gonna see if I can make a quick patch for it, if it's a lot of effort I might have to go with a different approach.

@MarkEWaite

This comment has been minimized.

Copy link

MarkEWaite replied Jan 8, 2015

Timeout on clone can be adjusted in the job definition from the job configuration page. Timeout on checkout can likewise be adjusted there. Timeout on submodule update is not implemented.

@tmertens

This comment has been minimized.

Copy link

tmertens replied Jan 8, 2015

The timeout is actually occurring on push with the git publisher.

@MarkEWaite

This comment has been minimized.

Copy link

MarkEWaite replied Jan 8, 2015

If you're seeing a timeout on push, that would usually indicate that the authentication information is not correct. Unless your Jenkins job is adding enormous content to the repository, the push should be a very fast operation.

@tmertens

This comment has been minimized.

Copy link

tmertens replied Jan 8, 2015

The timeout on push occurs because heroku builds all of the resources as part of the push, so for large web apps it can take up to 15 minutes. Looking at the implementation for the other git 'additional behaviours' it seems like it should be easy to implement a similar timeout override for the push command.

Please sign in to comment.