Skip to content

Commit

Permalink
Merge pull request #991 from coursier/topic/bootstrap-library
Browse files Browse the repository at this point in the history
Move bootstrap generation to separate library
  • Loading branch information
alexarchambault committed Dec 21, 2018
2 parents d848b57 + 1aaf23e commit f06127a
Show file tree
Hide file tree
Showing 23 changed files with 633 additions and 421 deletions.
28 changes: 18 additions & 10 deletions build.sbt
Expand Up @@ -131,7 +131,7 @@ lazy val cats = crossProject("interop", "cats")(JSPlatform, JVMPlatform)
lazy val catsJvm = cats.jvm
lazy val catsJs = cats.js

lazy val bootstrap = project("bootstrap")
lazy val `bootstrap-launcher` = project("bootstrap-launcher")
.enablePlugins(SbtProguard)
.settings(
pureJava,
Expand All @@ -142,6 +142,12 @@ lazy val bootstrap = project("bootstrap")
proguardedBootstrap
)

lazy val bootstrap = project("bootstrap")
.settings(
shared,
coursierPrefix
)

lazy val extra = project("extra")
.enablePlugins(ShadingPlugin)
.dependsOn(coreJvm, cacheJvm)
Expand Down Expand Up @@ -176,13 +182,13 @@ lazy val extra = project("extra")
)

lazy val cli = project("cli")
.dependsOn(coreJvm, cacheJvm, extra)
.dependsOn(bootstrap, coreJvm, cacheJvm, extra)
.enablePlugins(PackPlugin, SbtProguard)
.settings(
shared,
dontPublishIn("2.11"),
coursierPrefix,
unmanagedResources.in(Test) += proguardedJar.in(bootstrap).in(Compile).value,
unmanagedResources.in(Test) += proguardedJar.in(`bootstrap-launcher`).in(Compile).value,
scalacOptions += "-Ypartial-unification",
libs ++= {
if (scalaBinaryVersion.value == "2.12")
Expand Down Expand Up @@ -276,6 +282,7 @@ lazy val jvm = project("jvm")
cacheJvm,
scalazJvm,
catsJvm,
`bootstrap-launcher`,
bootstrap,
extra,
cli,
Expand Down Expand Up @@ -316,6 +323,7 @@ lazy val coursier = project("coursier")
paths,
cacheJvm,
cacheJs,
`bootstrap-launcher`,
bootstrap,
extra,
cli,
Expand All @@ -338,8 +346,8 @@ lazy val addBootstrapJarAsResource = {
import java.nio.file.Files

packageBin.in(Compile) := {
val originalBootstrapJar = packageBin.in(bootstrap).in(Compile).value
val bootstrapJar = proguardedJar.in(bootstrap).in(Compile).value
val originalBootstrapJar = packageBin.in(`bootstrap-launcher`).in(Compile).value
val bootstrapJar = proguardedJar.in(`bootstrap-launcher`).in(Compile).value
val source = packageBin.in(Compile).value

val dest = source.getParentFile / (source.getName.stripSuffix(".jar") + "-with-bootstrap.jar")
Expand All @@ -357,8 +365,8 @@ lazy val proguardedJarWithBootstrap = Def.task {

import java.nio.file.Files

val bootstrapJar = proguardedJar.in(bootstrap).in(Compile).value
val origBootstrapJar = packageBin.in(bootstrap).in(Compile).value
val bootstrapJar = proguardedJar.in(`bootstrap-launcher`).in(Compile).value
val origBootstrapJar = packageBin.in(`bootstrap-launcher`).in(Compile).value
val source = proguardedJar.value

val dest = source.getParentFile / (source.getName.stripSuffix(".jar") + "-with-bootstrap.jar")
Expand All @@ -379,9 +387,9 @@ lazy val proguardedBootstrap = Seq(
proguardVersion.in(Proguard) := SharedVersions.proguard,
proguardOptions.in(Proguard) ++= Seq(
"-dontwarn",
"-repackageclasses coursier.bootstrap",
"-keep class coursier.bootstrap.Bootstrap {\n public static void main(java.lang.String[]);\n}",
"-keep class coursier.bootstrap.IsolatedClassLoader {\n public java.lang.String[] getIsolationTargets();\n}"
"-repackageclasses coursier.bootstrap.launcher",
"-keep class coursier.bootstrap.launcher.Bootstrap {\n public static void main(java.lang.String[]);\n}",
"-keep class coursier.bootstrap.launcher.IsolatedClassLoader {\n public java.lang.String[] getIsolationTargets();\n}"
),
javaOptions.in(Proguard, proguard) := Seq("-Xmx3172M"),
artifactPath.in(Proguard) := proguardDirectory.in(Proguard).value / "bootstrap.jar"
Expand Down
@@ -1,4 +1,4 @@
package coursier.bootstrap;
package coursier.bootstrap.launcher;

import java.io.*;
import java.lang.reflect.InvocationTargetException;
Expand Down Expand Up @@ -37,7 +37,8 @@ static byte[] readFullySync(InputStream is) throws IOException {
return buffer.toByteArray();
}

final static String resourceDir = "coursier/bootstrap/";
final static String resourceDir = "coursier/bootstrap/launcher/";
final static String jarDir = resourceDir + "jars/";

final static String defaultURLResource = resourceDir + "bootstrap-jar-urls";
final static String defaultJarResource = resourceDir + "bootstrap-jar-resources";
Expand All @@ -55,28 +56,41 @@ static String[] readStringSequence(String resource) throws IOException {
return content.split("\n");
}

/**
*
* @param cacheDir can be null if nothing should be downloaded!
* @param isolationIDs
* @param bootstrapProtocol
* @param loader
* @return
* @throws IOException
*/
static Map<String, URL[]> readIsolationContexts(File cacheDir, String[] isolationIDs, String bootstrapProtocol, ClassLoader loader) throws IOException {
final Map<String, URL[]> perContextURLs = new LinkedHashMap<String, URL[]>();
static String readString(String resource) throws IOException {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
InputStream is = loader.getResourceAsStream(resource);
if (is == null)
return null;
byte[] rawContent = readFullySync(is);
return new String(rawContent, "UTF-8");
}

static ClassLoader readBaseLoaders(File cacheDir, ClassLoader baseLoader, BootstrapURLStreamHandlerFactory factory, ClassLoader loader) throws IOException {

ClassLoader parentLoader = baseLoader;
int i = 1;
while (true) {
String[] strUrls = readStringSequence(resourceDir + "bootstrap-jar-urls-" + i);
String[] resources = readStringSequence(resourceDir + "bootstrap-jar-resources-" + i);
String nameOrNull = readString(resourceDir + "bootstrap-loader-name-" + i);
String[] names;
if (nameOrNull == null)
names = new String[]{};
else
names = new String[]{ nameOrNull };

for (String isolationID: isolationIDs) {
String[] strUrls = readStringSequence(resourceDir + "bootstrap-isolation-" + isolationID + "-jar-urls");
String[] resources = readStringSequence(resourceDir + "bootstrap-isolation-" + isolationID + "-jar-resources");
List<URL> urls = getURLs(strUrls, resources, bootstrapProtocol, loader);
List<URL> localURLs = getLocalURLs(urls, cacheDir, bootstrapProtocol);
if (strUrls.length == 0 && resources.length == 0 && nameOrNull == null)
break;

perContextURLs.put(isolationID, localURLs.toArray(new URL[localURLs.size()]));
List<URL> urls = getURLs(strUrls, resources, factory, loader);
List<URL> localURLs = getLocalURLs(urls, cacheDir, factory);

parentLoader = new IsolatedClassLoader(localURLs.toArray(new URL[localURLs.size()]), parentLoader, names);

i = i + 1;
}

return perContextURLs;
return parentLoader;
}

final static int concurrentDownloadCount = 6;
Expand Down Expand Up @@ -158,11 +172,10 @@ static void writeBytesToFile(File file, byte[] bytes) throws IOException {
*
* @param urls
* @param cacheDir
* @param bootstrapProtocol
* @return
* @throws MalformedURLException
*/
static List<URL> getLocalURLs(List<URL> urls, final File cacheDir, String bootstrapProtocol) throws MalformedURLException {
static List<URL> getLocalURLs(List<URL> urls, final File cacheDir, BootstrapURLStreamHandlerFactory factory) throws MalformedURLException {

ThreadFactory threadFactory = new ThreadFactory() {
// from scalaz Strategy.DefaultDaemonThreadFactory
Expand All @@ -186,7 +199,7 @@ public Thread newThread(Runnable r) {

String protocol = url.getProtocol();

if (protocol.equals("file") || protocol.equals(bootstrapProtocol)) {
if (protocol.equals("file") || protocol.equals(factory.getProtocol())) {
localURLs.add(url);
} else {
// fourth argument is false because we don't want to store local files when bootstrapping
Expand Down Expand Up @@ -314,7 +327,7 @@ static void setExtraProperties(String resource) throws IOException {
}
}

static List<URL> getURLs(String[] rawURLs, String[] resources, String bootstrapProtocol, ClassLoader loader) throws MalformedURLException {
static List<URL> getURLs(String[] rawURLs, String[] resources, BootstrapURLStreamHandlerFactory factory, ClassLoader loader) throws MalformedURLException {

List<String> errors = new ArrayList<String>();
List<URL> urls = new ArrayList<URL>();
Expand All @@ -330,12 +343,12 @@ static List<URL> getURLs(String[] rawURLs, String[] resources, String bootstrapP
}

for (String resource : resources) {
URL url = loader.getResource(resource);
URL url = loader.getResource(jarDir + resource);
if (url == null) {
String message = "Resource " + resource + " not found";
errors.add(message);
} else {
URL url0 = new URL(bootstrapProtocol, null, resource);
URL url0 = new URL(factory.getProtocol(), null, resource);
urls.add(url0);
}
}
Expand All @@ -352,27 +365,6 @@ static List<URL> getURLs(String[] rawURLs, String[] resources, String bootstrapP
return urls;
}

// JARs from JARs can't be used directly, see:
// http://stackoverflow.com/questions/183292/classpath-including-jar-within-a-jar/2326775#2326775
// Loading them via a custom protocol, inspired by:
// http://stackoverflow.com/questions/26363573/registering-and-using-a-custom-java-net-url-protocol/26409796#26409796
static void registerBootstrapUnder(final String bootstrapProtocol, final ClassLoader loader) {
URL.setURLStreamHandlerFactory(new URLStreamHandlerFactory() {
public URLStreamHandler createURLStreamHandler(String protocol) {
return bootstrapProtocol.equals(protocol) ? new URLStreamHandler() {
protected URLConnection openConnection(URL url) throws IOException {
String path = url.getPath();
URL resURL = loader.getResource(path);
if (resURL == null)
throw new FileNotFoundException("Resource " + path);
return resURL.openConnection();
}
} : null;
}
});
}


public static void main(String[] args) throws Throwable {

setMainProperties(mainJarPath(), args);
Expand All @@ -382,28 +374,20 @@ public static void main(String[] args) throws Throwable {

File cacheDir = CachePath.defaultCacheDirectory();

Random rng = new Random();
String protocol = "bootstrap" + rng.nextLong();

ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();

registerBootstrapUnder(protocol, contextLoader);
BootstrapURLStreamHandlerFactory factory = new BootstrapURLStreamHandlerFactory(jarDir, contextLoader);

String[] strUrls = readStringSequence(defaultURLResource);
String[] resources = readStringSequence(defaultJarResource);
List<URL> urls = getURLs(strUrls, resources, protocol, contextLoader);
List<URL> localURLs = getLocalURLs(urls, cacheDir, protocol);
List<URL> urls = getURLs(strUrls, resources, factory, contextLoader);
List<URL> localURLs = getLocalURLs(urls, cacheDir, factory);

String[] isolationIDs = readStringSequence(isolationIDsResource);
Map<String, URL[]> perIsolationContextURLs = readIsolationContexts(cacheDir, isolationIDs, protocol, contextLoader);

Thread thread = Thread.currentThread();
ClassLoader parentClassLoader = thread.getContextClassLoader();

for (String isolationID: isolationIDs) {
URL[] contextURLs = perIsolationContextURLs.get(isolationID);
parentClassLoader = new IsolatedClassLoader(contextURLs, parentClassLoader, new String[]{ isolationID });
}
parentClassLoader = readBaseLoaders(cacheDir, parentClassLoader, factory, contextLoader);

ClassLoader classLoader = new URLClassLoader(localURLs.toArray(new URL[localURLs.size()]), parentClassLoader);

Expand Down
@@ -0,0 +1,59 @@
package coursier.bootstrap.launcher;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLStreamHandler;
import java.net.URLStreamHandlerFactory;
import java.util.Random;

// JARs from JARs can't be used directly, see:
// http://stackoverflow.com/questions/183292/classpath-including-jar-within-a-jar/2326775#2326775
// Loading them via a custom protocol, inspired by:
// http://stackoverflow.com/questions/26363573/registering-and-using-a-custom-java-net-url-protocol/26409796#26409796
public class BootstrapURLStreamHandlerFactory implements URLStreamHandlerFactory {
private final String bootstrapProtocol;
private final String basePath;
private final ClassLoader loader;
private boolean registered = false;

public BootstrapURLStreamHandlerFactory(String bootstrapProtocol, String basePath, ClassLoader loader) {
this.bootstrapProtocol = bootstrapProtocol;
this.basePath = basePath;
this.loader = loader;
}

public BootstrapURLStreamHandlerFactory(String basePath, ClassLoader loader) {
Random rng = new Random();
this.bootstrapProtocol = "bootstrap" + rng.nextLong();
this.basePath = basePath;
this.loader = loader;
}

public URLStreamHandler createURLStreamHandler(String protocol) {
return bootstrapProtocol.equals(protocol) ? new URLStreamHandler() {
protected URLConnection openConnection(URL url) throws IOException {
String path = url.getPath();
URL resURL = loader.getResource(basePath + path);
if (resURL == null)
throw new FileNotFoundException("Resource " + basePath + path);
return resURL.openConnection();
}
} : null;
}

public String getProtocol() {
registerIfNeeded();
return bootstrapProtocol;
}

public void register() {
URL.setURLStreamHandlerFactory(this);
registered = true;
}
public void registerIfNeeded() {
if (!registered)
register();
}
}
@@ -1,4 +1,4 @@
package coursier.bootstrap;
package coursier.bootstrap.launcher;

import java.net.URL;
import java.net.URLClassLoader;
Expand Down
6 changes: 6 additions & 0 deletions modules/bootstrap/BUILD
@@ -0,0 +1,6 @@
scala_library(
name = "bootstrap",
dependencies = [
],
sources = rglobs("*.scala"),
)

0 comments on commit f06127a

Please sign in to comment.