Skip to content

Commit

Permalink
[JENKINS-62014] Add build step environment filters (#4683)
Browse files Browse the repository at this point in the history 
Co-authored-by: Wadeck Follonier <wadeck.follonier@gmail.com>
Co-authored-by: Daniel Beck <daniel-beck@users.noreply.github.com>
  • Loading branch information
@daniel-beck @Wadeck
3 people committed Jul 19, 2020
1 parent fb7f890 commit 0134fb1
Show file tree
Hide file tree
Showing 28 changed files with 1,615 additions and 17 deletions.
84 changes: 79 additions & 5 deletions core/src/main/java/hudson/Launcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import hudson.Proc.LocalProc;
import hudson.model.Computer;
import jenkins.util.MemoryReductionUtil;
import hudson.model.Run;
import hudson.util.QuotedStringTokenizer;
import jenkins.model.Jenkins;
import hudson.model.TaskListener;
Expand All @@ -39,8 +40,12 @@
import hudson.util.ArgumentListBuilder;
import hudson.util.ProcessTree;
import jenkins.security.MasterToSlaveCallable;
import jenkins.tasks.filters.EnvVarsFilterRuleWrapper;
import jenkins.tasks.filters.EnvVarsFilterLocalRule;
import jenkins.tasks.filters.EnvVarsFilterableBuilder;
import org.apache.commons.io.input.NullInputStream;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.Beta;
import org.kohsuke.accmod.restrictions.NoExternalUse;

import edu.umd.cs.findbugs.annotations.CheckForNull;
Expand Down Expand Up @@ -90,6 +95,9 @@ public abstract class Launcher {
@CheckForNull
protected final VirtualChannel channel;

@Restricted(Beta.class)
protected EnvVarsFilterRuleWrapper envVarsFilterRuleWrapper;

public Launcher(@NonNull TaskListener listener, @CheckForNull VirtualChannel channel) {
this.listener = listener;
this.channel = channel;
Expand All @@ -103,6 +111,25 @@ protected Launcher(@NonNull Launcher launcher) {
this(launcher.listener, launcher.channel);
}

/**
* Build the environment filter rules that will be applied on the environment variables
* @param run The run that requested the command interpretation, could be <code>null</code> if outside of a run context.
* @param builder The builder that asked to run this command
*
* @since TODO
*/
@Restricted(Beta.class)
public void prepareFilterRules(@CheckForNull Run<?,?> run, @NonNull EnvVarsFilterableBuilder builder){
List<EnvVarsFilterLocalRule> specificRuleList = builder.buildEnvVarsFilterRules();
EnvVarsFilterRuleWrapper ruleWrapper = EnvVarsFilterRuleWrapper.createRuleWrapper(run, builder, this, specificRuleList);
this.setEnvVarsFilterRuleWrapper(ruleWrapper);
}

@Restricted(Beta.class)
protected void setEnvVarsFilterRuleWrapper(EnvVarsFilterRuleWrapper envVarsFilterRuleWrapper) {
this.envVarsFilterRuleWrapper = envVarsFilterRuleWrapper;
}

/**
* Gets the channel that can be used to run a program remotely.
*
Expand Down Expand Up @@ -171,6 +198,13 @@ public final class ProcStarter {
protected InputStream stdin = NULL_INPUT_STREAM;
@CheckForNull
protected String[] envs = null;
/**
* Represent the build step, either from legacy build process or from pipeline one
*/
@CheckForNull
@Restricted(Beta.class)
protected EnvVarsFilterableBuilder envVarsFilterableBuilder = null;

/**
* True to reverse the I/O direction.
*
Expand Down Expand Up @@ -446,6 +480,26 @@ public ProcStarter writeStdin() {
return this;
}

/**
* Specify the build step that want to run the command to enable the environment filters
* @return {@code this}
* @since TODO
*/
@Restricted(Beta.class)
public ProcStarter buildStep(EnvVarsFilterableBuilder envVarsFilterableBuilder){
this.envVarsFilterableBuilder = envVarsFilterableBuilder;
return this;
}

/**
* @return if set, returns the build step that wants to run the command
* @since TODO
*/
@Restricted(Beta.class)
public @CheckForNull
EnvVarsFilterableBuilder buildStep() {
return envVarsFilterableBuilder;
}

/**
* Starts the new process as configured.
Expand Down Expand Up @@ -494,7 +548,7 @@ public int join() throws IOException, InterruptedException {
*/
@NonNull
public ProcStarter copy() {
ProcStarter rhs = new ProcStarter().cmds(commands).pwd(pwd).masks(masks).stdin(stdin).stdout(stdout).stderr(stderr).envs(envs).quiet(quiet);
ProcStarter rhs = new ProcStarter().cmds(commands).pwd(pwd).masks(masks).stdin(stdin).stdout(stdout).stderr(stderr).envs(envs).quiet(quiet).buildStep(envVarsFilterableBuilder);
rhs.stdoutListener = stdoutListener;
rhs.reverseStdin = this.reverseStdin;
rhs.reverseStderr = this.reverseStderr;
Expand Down Expand Up @@ -924,6 +978,12 @@ public Proc launch(ProcStarter ps) throws IOException {

EnvVars jobEnv = inherit(ps.envs);

if (envVarsFilterRuleWrapper != null) {
envVarsFilterRuleWrapper.filter(jobEnv, this, listener);
// reset the rules to prevent build step without rules configuration to re-use those
envVarsFilterRuleWrapper = null;
}

// replace variables in command line
String[] jobCmd = new String[ps.commands.size()];
for ( int idx = 0 ; idx < jobCmd.length; idx++ )
Expand Down Expand Up @@ -1054,7 +1114,11 @@ public Proc launch(ProcStarter ps) throws IOException {
final String workDir = psPwd==null ? null : psPwd.getRemote();

try {
return new ProcImpl(getChannel().call(new RemoteLaunchCallable(ps.commands, ps.masks, ps.envs, in, ps.reverseStdin, out, ps.reverseStdout, err, ps.reverseStderr, ps.quiet, workDir, listener, ps.stdoutListener)));
RemoteLaunchCallable remote = new RemoteLaunchCallable(ps.commands, ps.masks, ps.envs, in, ps.reverseStdin, out, ps.reverseStdout, err, ps.reverseStderr, ps.quiet, workDir, listener, ps.stdoutListener);
remote.setEnvVarsFilterRuleWrapper(envVarsFilterRuleWrapper);
// reset the rules to prevent build step without rules configuration to re-use those
envVarsFilterRuleWrapper = null;
return new ProcImpl(getChannel().call(remote));
} catch (InterruptedException e) {
throw (IOException)new InterruptedIOException().initCause(e);
}
Expand Down Expand Up @@ -1279,8 +1343,10 @@ private static class RemoteLaunchCallable extends MasterToSlaveCallable<RemotePr
private final boolean reverseStdin, reverseStdout, reverseStderr;
private final boolean quiet;

RemoteLaunchCallable(@NonNull List<String> cmd, @CheckForNull boolean[] masks, @CheckForNull String[] env,
@CheckForNull InputStream in, boolean reverseStdin,
private EnvVarsFilterRuleWrapper envVarsFilterRuleWrapper;

RemoteLaunchCallable(@NonNull List<String> cmd, @CheckForNull boolean[] masks, @CheckForNull String[] env,
@CheckForNull InputStream in, boolean reverseStdin,
@CheckForNull OutputStream out, boolean reverseStdout,
@CheckForNull OutputStream err, boolean reverseStderr,
boolean quiet, @CheckForNull String workDir, @NonNull TaskListener listener, @CheckForNull TaskListener stdoutListener) {
Expand All @@ -1299,9 +1365,17 @@ private static class RemoteLaunchCallable extends MasterToSlaveCallable<RemotePr
this.quiet = quiet;
}

@Restricted(NoExternalUse.class)
public void setEnvVarsFilterRuleWrapper(EnvVarsFilterRuleWrapper envVarsFilterRuleWrapper) {
this.envVarsFilterRuleWrapper = envVarsFilterRuleWrapper;
}

public RemoteProcess call() throws IOException {
final Channel channel = getOpenChannelOrFail();
Launcher.ProcStarter ps = new LocalLauncher(listener).launch();
LocalLauncher localLauncher = new LocalLauncher(listener);
localLauncher.setEnvVarsFilterRuleWrapper(envVarsFilterRuleWrapper);

Launcher.ProcStarter ps = localLauncher.launch();
ps.cmds(cmd).masks(masks).envs(env).stdin(in).stderr(err).quiet(quiet);
if (stdoutListener != null) {
ps.stdout(stdoutListener.getLogger());
Expand Down
28 changes: 26 additions & 2 deletions core/src/main/java/hudson/tasks/BatchFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,19 @@
import hudson.model.AbstractProject;
import hudson.util.FormValidation;
import hudson.util.LineEndingConversion;
import jenkins.tasks.filters.EnvVarsFilterLocalRule;
import jenkins.tasks.filters.EnvVarsFilterLocalRuleDescriptor;
import org.jenkinsci.Symbol;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.Beta;
import org.kohsuke.accmod.restrictions.DoNotUse;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.DataBoundSetter;
import org.kohsuke.stapler.QueryParameter;

import java.io.ObjectStreamException;
import java.util.ArrayList;
import java.util.List;

import edu.umd.cs.findbugs.annotations.CheckForNull;

Expand All @@ -51,6 +56,17 @@ public BatchFile(String command) {
super(LineEndingConversion.convertEOL(command, LineEndingConversion.EOLType.Windows));
}

/**
* Set local environment variable filter rules
* @param configuredLocalRules list of local environment filter rules
* @since TODO
*/
@Restricted(Beta.class)
@DataBoundSetter
public void setConfiguredLocalRules(List<EnvVarsFilterLocalRule> configuredLocalRules) {
this.configuredLocalRules = configuredLocalRules;
}

private Integer unstableReturn;

public String[] buildCommandLine(FilePath script) {
Expand Down Expand Up @@ -80,9 +96,11 @@ protected boolean isErrorlevelForUnstableBuild(int exitCode) {
return this.unstableReturn != null && exitCode != 0 && this.unstableReturn.equals(exitCode);
}

private Object readResolve() throws ObjectStreamException {
private Object readResolve() {
BatchFile batch = new BatchFile(command);
batch.setUnstableReturn(unstableReturn);
// backward compatibility
batch.setConfiguredLocalRules(configuredLocalRules == null ? new ArrayList<>() : configuredLocalRules);
return batch;
}

Expand Down Expand Up @@ -124,5 +142,11 @@ public FormValidation doCheckUnstableReturn(@QueryParameter String value) {
public boolean isApplicable(Class<? extends AbstractProject> jobType) {
return true;
}

// used by Jelly view
@Restricted(NoExternalUse.class)
public List<EnvVarsFilterLocalRuleDescriptor> getApplicableLocalRules() {
return EnvVarsFilterLocalRuleDescriptor.allApplicableFor(BatchFile.class);
}
}
}
42 changes: 40 additions & 2 deletions core/src/main/java/hudson/tasks/CommandInterpreter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,16 @@
import hudson.model.Result;
import hudson.model.TaskListener;
import hudson.remoting.ChannelClosedException;
import jenkins.tasks.filters.EnvVarsFilterLocalRule;
import jenkins.tasks.filters.EnvVarsFilterableBuilder;
import jenkins.tasks.filters.EnvVarsFilterException;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.Beta;
import org.kohsuke.accmod.restrictions.NoExternalUse;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand All @@ -47,12 +55,18 @@
*
* @author Kohsuke Kawaguchi
*/
public abstract class CommandInterpreter extends Builder {
public abstract class CommandInterpreter extends Builder implements EnvVarsFilterableBuilder {
/**
* Command to execute. The format depends on the actual {@link CommandInterpreter} implementation.
*/
protected final String command;

/**
* List of configured environment filter rules
*/
@Restricted(Beta.class)
protected List<EnvVarsFilterLocalRule> configuredLocalRules = new ArrayList<>();

public CommandInterpreter(String command) {
this.command = command;
}
Expand All @@ -61,6 +75,16 @@ public final String getCommand() {
return command;
}

public @NonNull List<EnvVarsFilterLocalRule> buildEnvVarsFilterRules() {
return new ArrayList<>(configuredLocalRules);
}

// used by Jelly view
@Restricted(NoExternalUse.class)
public List<EnvVarsFilterLocalRule> getConfiguredLocalRules() {
return configuredLocalRules;
}

@Override
public boolean perform(AbstractBuild<?,?> build, Launcher launcher, BuildListener listener) throws InterruptedException {
return perform(build,launcher,(TaskListener)listener);
Expand Down Expand Up @@ -106,7 +130,21 @@ public boolean perform(AbstractBuild<?,?> build, Launcher launcher, TaskListener
for(Map.Entry<String,String> e : build.getBuildVariables().entrySet())
envVars.put(e.getKey(),e.getValue());

r = join(launcher.launch().cmds(buildCommandLine(script)).envs(envVars).stdout(listener).pwd(ws).start());
launcher.prepareFilterRules(build, this);

Launcher.ProcStarter procStarter = launcher.launch();
procStarter.cmds(buildCommandLine(script))
.envs(envVars)
.stdout(listener)
.pwd(ws);

try {
Proc proc = procStarter.start();
r = join(proc);
} catch (EnvVarsFilterException se) {
LOGGER.log(Level.FINE, "Environment variable filtering failed", se);
return false;
}

if(isErrorlevelForUnstableBuild(r)) {
build.setResult(Result.UNSTABLE);
Expand Down
29 changes: 25 additions & 4 deletions core/src/main/java/hudson/tasks/Shell.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,18 @@
import hudson.remoting.VirtualChannel;
import hudson.util.FormValidation;
import java.io.IOException;
import java.io.ObjectStreamException;

import hudson.util.LineEndingConversion;
import jenkins.security.MasterToSlaveCallable;
import jenkins.tasks.filters.EnvVarsFilterLocalRule;
import jenkins.tasks.filters.EnvVarsFilterLocalRuleDescriptor;
import net.sf.json.JSONObject;
import org.apache.commons.lang.SystemUtils;
import org.jenkinsci.Symbol;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.Beta;
import org.kohsuke.accmod.restrictions.DoNotUse;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.DataBoundSetter;
import org.kohsuke.stapler.StaplerRequest;
Expand All @@ -64,9 +68,18 @@ public Shell(String command) {
super(LineEndingConversion.convertEOL(command, LineEndingConversion.EOLType.Unix));
}

private Integer unstableReturn;

/**
* Set local environment variable filter rules
* @param configuredLocalRules list of local environment filter rules
* @since TODO
*/
@Restricted(Beta.class)
@DataBoundSetter
public void setConfiguredLocalRules(List<EnvVarsFilterLocalRule> configuredLocalRules) {
this.configuredLocalRules = configuredLocalRules;
}

private Integer unstableReturn;

/**
* Older versions of bash have a bug where non-ASCII on the first line
Expand Down Expand Up @@ -124,9 +137,11 @@ public DescriptorImpl getDescriptor() {
return (DescriptorImpl)super.getDescriptor();
}

private Object readResolve() throws ObjectStreamException {
private Object readResolve() {
Shell shell = new Shell(command);
shell.setUnstableReturn(unstableReturn);
// backward compatibility
shell.setConfiguredLocalRules(configuredLocalRules == null ? new ArrayList<>() : configuredLocalRules);
return shell;
}

Expand All @@ -141,6 +156,12 @@ public boolean isApplicable(Class<? extends AbstractProject> jobType) {
return true;
}

// used by Jelly view
@Restricted(NoExternalUse.class)
public List<EnvVarsFilterLocalRuleDescriptor> getApplicableLocalRules() {
return EnvVarsFilterLocalRuleDescriptor.allApplicableFor(Shell.class);
}

public String getShell() {
return shell;
}
Expand Down

0 comments on commit 0134fb1

Please sign in to comment.