Merge pull request #33 from jenkinsci/trusted-classloader

[JENKINS-34650] Added a trusted classloader that runs CPS code outside sandbox

doc/classloader.md

@@ -0,0 +1,49 @@
+# Class loading for CPS Groovy execution
+Pipeline Script needs to be loaded into JVM for it to run. This is done
+by a `GroovyClassLoader` created inside `CpsGroovyShell`.
+
+We create two classloaders in the following hierarchy
+
+    <<Jenkins UberClassLoader>> <-- <<trusted classloader>> <-- <<regular classloader>>
+
+Jenkins uber classloader (`PluginManager.uberClassLoader`) defines
+visibility to every code in every plugins, so that every domain model
+of Jenkins can be accessed. This is the starting point of the hierarchy.
+
+"Trusted classloader" (TCL) and "regular classloader" (RCL) are both
+`GroovyClassLoader`. Groovy compiler associated with them are configured for Jenkins Pipeline.
+For example, scripts loaded by those are CPS transformed to run with groovy-cps.
+See `CpsGroovyShellFactory` for more about the customizations that happen here.
+
+Additionally, scripts loaded in RCL lives in the security sandbox
+(unless the user opts out of it, in which case it requires approval.)
+This classloader is meant to be used to load user-written Groovy scripts,
+which is not trusted.
+
+Scripts loaded in TCL, OTOH, does not live in the security sandbox. This
+classloader is meant to be used to load Groovy code packaged inside
+plugins and global libraries. Write access to these sources should be
+restricted to `RUN_SCRIPTS` permission.
+
+## Persisting code & surviving restarts
+When a Groovy script is loaded via one of `GroovyShell.parse*()` and
+`eval*()` methods, script text is captured and persisted as a part
+of the program state. This ensures that the exact same code is available
+when a pipeline execution resumes execution after a JVM restart.
+
+This behaviour is desirable for the situation where script is supplied
+from outside and not readily available at the point of restart.
+For example, `Jenkinsfile` and scripts loaded from the `load` step
+fits this description.
+
+It's also possible to augument `GroovyClassLoader` classpath via
+`addURL()` so that scripts are loaded on-demand whenever needed.
+This is more suitable for scripts that are considered a part of the
+system, such as global libraries or plugin code.
+
+Those scripts are not snapshotted & persisted with the running program,
+so if they change while the program is running, with or without Jenkins restarts
+in the middle, then the program could fail. (For example, if a program
+call stack includes a class from a global library and that class goes away,
+then the program fails to survive the restart because the call stack cannot
+be deserialized.

pom.xml

@@ -171,7 +171,7 @@
         <dependency>
             <groupId>com.cloudbees</groupId>
             <artifactId>groovy-cps</artifactId>
-            <version>1.6</version>
+            <version>1.8-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.ui</groupId>

src/main/java/org/jenkinsci/plugins/workflow/cps/CpsFlowDefinition.java

@@ -48,6 +48,7 @@
 import org.jenkinsci.plugins.scriptsecurity.scripts.languages.GroovyLanguage;
 
 import static org.jenkinsci.plugins.workflow.cps.persistence.PersistenceContext.*;
+
 import org.kohsuke.stapler.QueryParameter;
 import org.kohsuke.stapler.Stapler;
 import org.kohsuke.stapler.StaplerRequest;
@@ -127,7 +128,8 @@ public JSON doCheckScriptCompile(@QueryParameter String value) {
                 return CpsFlowDefinitionValidator.CheckStatus.SUCCESS.asJSON();
             }
             try {
-                new CpsGroovyShell(null).getClassLoader().parseClass(value);
+                CpsGroovyShell trusted = new CpsGroovyShellFactory(null).forTrusted().build();
+                new CpsGroovyShellFactory(null).withParent(trusted).build().getClassLoader().parseClass(value);
             } catch (CompilationFailedException x) {
                 return JSONArray.fromObject(CpsFlowDefinitionValidator.toCheckStatus(x).toArray());
             }

src/main/java/org/jenkinsci/plugins/workflow/cps/CpsFlowExecution.java

@@ -270,6 +270,16 @@ public class CpsFlowExecution extends FlowExecution {
      * It is reset to null after completion.
      */
     private transient CpsGroovyShell shell;
+
+    /**
+     * Groovy compiler wih CPS transformation but not sandbox.
+     * Used by plugins to insert code that runs outside sandbox.
+     *
+     * By the time the script starts running, this field is set to non-null.
+     * It is reset to null after completion.
+     */
+    private transient CpsGroovyShell trusted;
+
     /** Class of the {@link CpsScript}; its loader is a {@link groovy.lang.GroovyClassLoader.InnerLoader}, not the same as {@code shell.getClassLoader()}. */
     private transient Class<?> scriptClass;
 
@@ -302,12 +312,22 @@ private Object readResolve() {
     /**
      * Returns a groovy compiler used to load the script.
      *
+     * @see "doc/classloader.md"
      * @see GroovyShell#getClassLoader()
      */
     public GroovyShell getShell() {
         return shell;
     }
 
+    /**
+     * Returns a groovy compiler used to load the trusted script.
+     *
+     * @see "doc/classloader.md"
+     */
+    public GroovyShell getTrustedShell() {
+        return trusted;
+    }
+
     public FlowNodeStorage getStorage() {
         return storage;
     }
@@ -378,7 +398,10 @@ private Env createInitialEnv() {
     }
 
     private CpsScript parseScript() throws IOException {
-        shell = new CpsGroovyShell(this);
+        // classloader hierarchy. See doc/classloader.md
+        trusted = new CpsGroovyShellFactory(this).forTrusted().build();
+        shell = new CpsGroovyShellFactory(this).withParent(trusted).build();
+
         CpsScript s = (CpsScript) shell.reparse("WorkflowScript",script);
 
         for (Entry<String, String> e : loadedScripts.entrySet()) {
@@ -864,6 +887,7 @@ public boolean isComplete() {
 
         // clean up heap
         shell = null;
+        trusted = null;
         SerializableClassRegistry.getInstance().release(scriptClass.getClassLoader());
         Introspector.flushFromCaches(scriptClass); // does not handle other derived script classes, but this is only SoftReference anyway
         scriptClass = null;

src/main/java/org/jenkinsci/plugins/workflow/cps/CpsGroovyShell.java

@@ -1,26 +1,21 @@
 package org.jenkinsci.plugins.workflow.cps;
 
-import com.cloudbees.groovy.cps.CpsTransformer;
-import com.cloudbees.groovy.cps.NonCPS;
-import com.cloudbees.groovy.cps.SandboxCpsTransformer;
-import com.cloudbees.groovy.cps.TransformerConfiguration;
 import groovy.lang.Binding;
 import groovy.lang.GroovyCodeSource;
 import groovy.lang.GroovyShell;
 import groovy.lang.Script;
-import jenkins.model.Jenkins;
 import org.codehaus.groovy.control.CompilationFailedException;
 import org.codehaus.groovy.control.CompilerConfiguration;
-import org.codehaus.groovy.control.customizers.ImportCustomizer;
-import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.GroovySandbox;
 
-import java.io.IOException;
 import javax.annotation.CheckForNull;
+import java.io.IOException;
 
 /**
  * {@link GroovyShell} with additional tweaks necessary to run {@link CpsScript}
  *
  * @author Kohsuke Kawaguchi
+ * @see "doc/clasloader.md"
+ * @see CpsGroovyShellFactory
  */
 class CpsGroovyShell extends GroovyShell {
     /**
@@ -30,46 +25,12 @@ class CpsGroovyShell extends GroovyShell {
      */
     private final @CheckForNull CpsFlowExecution execution;
 
-    CpsGroovyShell(@CheckForNull CpsFlowExecution execution) {
-        super(makeClassLoader(),new Binding(),makeConfig(execution));
+    /**
+     * Use {@link CpsGroovyShellFactory} to instantiate it.
+     */
+    CpsGroovyShell(ClassLoader parent, @CheckForNull CpsFlowExecution execution, CompilerConfiguration cc) {
+        super(parent,new Binding(),cc);
         this.execution = execution;
-
-        for (GroovyShellDecorator d : GroovyShellDecorator.all()) {
-            d.configureShell(execution,this);
-        }
-    }
-
-    private static ClassLoader makeClassLoader() {
-        Jenkins j = Jenkins.getInstance();
-        ClassLoader cl = j != null ? j.getPluginManager().uberClassLoader : CpsGroovyShell.class.getClassLoader();
-        return GroovySandbox.createSecureClassLoader(cl);
-    }
-
-    private static CompilerConfiguration makeConfig(@CheckForNull CpsFlowExecution execution) {
-        ImportCustomizer ic = new ImportCustomizer();
-        ic.addStarImports(NonCPS.class.getPackage().getName());
-        ic.addStarImports("hudson.model","jenkins.model");
-
-        for (GroovyShellDecorator d : GroovyShellDecorator.all()) {
-            d.customizeImports(execution,ic);
-        }
-
-        CompilerConfiguration cc = new CompilerConfiguration();
-        cc.addCompilationCustomizers(ic);
-        cc.addCompilationCustomizers(makeCpsTransformer(execution));
-        cc.setScriptBaseClass(CpsScript.class.getName());
-
-        for (GroovyShellDecorator d : GroovyShellDecorator.all()) {
-            d.configureCompiler(execution,cc);
-        }
-
-        return cc;
-    }
-
-    private static CpsTransformer makeCpsTransformer(CpsFlowExecution execution) {
-        CpsTransformer t = (execution!=null && execution.isSandbox()) ? new SandboxCpsTransformer() : new CpsTransformer();
-        t.setConfiguration(new TransformerConfiguration().withClosureType(CpsClosure2.class));
-        return t;
     }
 
     public void prepareScript(Script script) {

src/main/java/org/jenkinsci/plugins/workflow/cps/CpsGroovyShellFactory.java

@@ -0,0 +1,128 @@
+package org.jenkinsci.plugins.workflow.cps;
+
+import com.cloudbees.groovy.cps.CpsTransformer;
+import com.cloudbees.groovy.cps.NonCPS;
+import com.cloudbees.groovy.cps.SandboxCpsTransformer;
+import com.cloudbees.groovy.cps.TransformerConfiguration;
+import groovy.lang.GroovyShell;
+import jenkins.model.Jenkins;
+import org.codehaus.groovy.control.CompilerConfiguration;
+import org.codehaus.groovy.control.customizers.ImportCustomizer;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.GroovySandbox;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Instantiates {@link CpsGroovyShell}.
+ *
+ * @author Kohsuke Kawaguchi
+ */
+class CpsGroovyShellFactory {
+    private final @CheckForNull CpsFlowExecution execution;
+    private boolean sandbox;
+    private List<GroovyShellDecorator> decorators;
+    private ClassLoader parent;
+
+    /**
+     * @param execution
+     *      Instantiated {@link CpsGroovyShell} will be used to load scripts for this execution.
+     */
+    public CpsGroovyShellFactory(@Nullable CpsFlowExecution execution) {
+        this.execution = execution;
+        this.sandbox = execution!=null && execution.isSandbox();
+        this.decorators = GroovyShellDecorator.all();
+    }
+
+    private CpsGroovyShellFactory(CpsFlowExecution execution, boolean sandbox, ClassLoader parent, List<GroovyShellDecorator> decorators) {
+        this.execution = execution;
+        this.sandbox = sandbox;
+        this.parent = parent;
+        this.decorators = decorators;
+    }
+
+    /**
+     * Derives a new factory for creating trusted {@link CpsGroovyShell}
+     */
+    public CpsGroovyShellFactory forTrusted() {
+        List<GroovyShellDecorator> inner = new ArrayList<>();
+        for (GroovyShellDecorator d : decorators) {
+            inner.add(d.forTrusted());
+        }
+        return new CpsGroovyShellFactory(execution, false, parent, inner);
+    }
+
+    /**
+     * Enables/disables the use of script-security on scripts loaded into {@link CpsGroovyShell}.
+     * This method can be called to override the setting in {@link CpsFlowExecution}.
+     */
+    public CpsGroovyShellFactory withSandbox(boolean b) {
+        this.sandbox = b;
+        return this;
+    }
+
+    public CpsGroovyShellFactory withParent(GroovyShell parent) {
+        return withParent(parent.getClassLoader());
+    }
+
+    /**
+     * Sets the parent classloader for the {@link CpsGroovyShell}.
+     */
+    public CpsGroovyShellFactory withParent(ClassLoader parent) {
+        this.parent = parent;
+        return this;
+    }
+
+    private CpsTransformer makeCpsTransformer() {
+        CpsTransformer t = sandbox ? new SandboxCpsTransformer() : new CpsTransformer();
+        t.setConfiguration(new TransformerConfiguration().withClosureType(CpsClosure2.class));
+        return t;
+    }
+
+    private CompilerConfiguration makeConfig() {
+        CompilerConfiguration cc = new CompilerConfiguration();
+
+        cc.addCompilationCustomizers(makeImportCustomizer());
+        cc.addCompilationCustomizers(makeCpsTransformer());
+        cc.setScriptBaseClass(CpsScript.class.getName());
+
+        for (GroovyShellDecorator d : decorators) {
+            d.configureCompiler(execution,cc);
+        }
+
+        return cc;
+    }
+
+    private ImportCustomizer makeImportCustomizer() {
+        ImportCustomizer ic = new ImportCustomizer();
+        ic.addStarImports(NonCPS.class.getPackage().getName());
+        ic.addStarImports("hudson.model","jenkins.model");
+
+        for (GroovyShellDecorator d : decorators) {
+            d.customizeImports(execution,ic);
+        }
+        return ic;
+    }
+
+    private ClassLoader makeClassLoader() {
+        Jenkins j = Jenkins.getInstance();
+        ClassLoader cl = j != null ? j.getPluginManager().uberClassLoader : CpsGroovyShell.class.getClassLoader();
+        return GroovySandbox.createSecureClassLoader(cl);
+    }
+
+    public CpsGroovyShell build() {
+        ClassLoader parent = this.parent;
+        if (parent==null)
+            parent = makeClassLoader();
+
+        CpsGroovyShell shell = new CpsGroovyShell(parent, execution, makeConfig());
+
+        for (GroovyShellDecorator d : decorators) {
+            d.configureShell(execution,shell);
+        }
+
+        return shell;
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/cps/GroovyClassLoaderWhitelist.java

@@ -9,6 +9,10 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Member;
 import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+
 import org.codehaus.groovy.runtime.DefaultGroovyMethods;
 import org.jenkinsci.plugins.scriptsecurity.sandbox.RejectedAccessException;
 import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.EnumeratingWhitelist;
@@ -21,7 +25,7 @@
  */
 class GroovyClassLoaderWhitelist extends Whitelist {
 
-    private final ClassLoader scriptLoader;
+    private final Collection<ClassLoader> scriptLoaders;
 
     /**
      * {@link ProxyWhitelist} has an optimization which bypasses {@code permits*} calls
@@ -31,17 +35,17 @@ class GroovyClassLoaderWhitelist extends Whitelist {
      */
     private final Whitelist delegate;
 
-    GroovyClassLoaderWhitelist(GroovyClassLoader scriptLoader, Whitelist delegate) {
-        this.scriptLoader = scriptLoader;
+    GroovyClassLoaderWhitelist(Whitelist delegate, GroovyClassLoader... scriptLoaders) {
+        this.scriptLoaders = Arrays.<ClassLoader>asList(scriptLoaders);
         this.delegate = delegate;
     }
 
     private boolean permits(Class<?> declaringClass) {
         ClassLoader cl = declaringClass.getClassLoader();
         if (cl instanceof GroovyClassLoader.InnerLoader) {
-            return cl.getParent()==scriptLoader;
+            return scriptLoaders.contains(cl.getParent());
         }
-        return cl == scriptLoader;
+        return scriptLoaders.contains(cl);
     }
 
     @Override public boolean permitsMethod(Method method, Object receiver, Object[] args) {