Skip to content

Commit

Permalink
[ZEPPELIN-18] Running pyspark without deploying python libraries to e…
Browse files Browse the repository at this point in the history
…very yarn node

- Fixed checkstyle
  • Loading branch information
jongyoul committed Jul 4, 2015
1 parent 929333d commit 94bdf30
Showing 1 changed file with 6 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -269,20 +269,20 @@ public SparkContext createSparkContext() {
conf.set(key, val);
}
}

//TODO(jongyoul): Move these codes into PySparkInterpreter.java
String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../");
File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib");
String[] pythonLibs = new String[] {"pyspark.zip", "py4j-0.8.2.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib: pythonLibs) {
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(zeppelinPythonLibPath, lib);
if(libFile.exists()) {
if (libFile.exists()) {
pythonLibUris.add(libFile.toURI().toString());
}
}
pythonLibUris.trimToSize();
if(pythonLibs.length == pythonLibUris.size()) {
if (pythonLibs.length == pythonLibUris.size()) {
conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris));
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
Expand Down

0 comments on commit 94bdf30

Please sign in to comment.