From 56fab25ff9f01014e5666f46b121ce65c8a07cb2 Mon Sep 17 00:00:00 2001 From: Andrea Peruffo Date: Fri, 17 Mar 2017 10:59:44 +0100 Subject: [PATCH] [ZEPPELIN-2278] Env variable to configure Npm registry --- conf/zeppelin-env.cmd.template | 1 + conf/zeppelin-env.sh.template | 1 + conf/zeppelin-site.xml.template | 6 ++++++ docs/install/configuration.md | 8 +++++++- .../apache/zeppelin/server/ZeppelinServer.java | 3 ++- .../zeppelin/conf/ZeppelinConfiguration.java | 5 +++++ .../zeppelin/helium/HeliumBundleFactory.java | 16 ++++++++++++---- .../zeppelin/helium/HeliumBundleFactoryTest.java | 8 +++++++- 8 files changed, 41 insertions(+), 7 deletions(-) diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index 548a7984c3a..cd1541e72c4 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -39,6 +39,7 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading +REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth? diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 5e32b15b8cf..754c8ef349e 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -44,6 +44,7 @@ # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading +# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index d1729424c35..6ce764b849c 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -251,6 +251,12 @@ Local repository for dependency loader + + zeppelin.helium.npm.registry + http://registry.npmjs.org/ + Remote Npm registry for Helium dependency loader + + zeppelin.interpreters org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter diff --git a/docs/install/configuration.md b/docs/install/configuration.md index ac41e6f202a..a35eb91594c 100644 --- a/docs/install/configuration.md +++ b/docs/install/configuration.md @@ -44,7 +44,7 @@ If both are defined, then the **environment variables** will take priority.
zeppelin.server.port
8080 Zeppelin server port
- Note: Please make sure you're not using the same port with + Note: Please make sure you're not using the same port with Zeppelin web application development port (default: 9000). @@ -281,6 +281,12 @@ If both are defined, then the **environment variables** will take priority. local-repo Local repository for dependency loader.
ex)visualiztion modules of npm. + +
ZEPPELIN_HELIUM_NPM_REGISTRY
+
zeppelin.helium.npm.registry
+ http://registry.npmjs.org/ + Remote Npm registry for Helium dependency loader +
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE
zeppelin.websocket.max.text.message.size
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 768a154cba0..ce38e46f046 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -114,12 +114,14 @@ public ZeppelinServer() throws Exception { * packaged into binary package. */ heliumBundleFactory = new HeliumBundleFactory( + conf, new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)), new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")), new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")), new File(conf.getRelativeDir("lib/node_modules/zeppelin-spell"))); } else { heliumBundleFactory = new HeliumBundleFactory( + conf, new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)), new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")), new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")), @@ -410,4 +412,3 @@ private static boolean isBinaryPackage(ZeppelinConfiguration conf) { return !new File(conf.getRelativeDir("zeppelin-web")).isDirectory(); } } - diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index facf89a96f7..55f502c3cfd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -424,6 +424,10 @@ public String getHeliumRegistry() { return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY); } + public String getHeliumNpmRegistry() { + return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY); + } + public String getNotebookAuthorizationPath() { return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir())); } @@ -633,6 +637,7 @@ public static enum ConfVars { ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL), + ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java index e04f9ac0298..c7765063384 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java @@ -33,6 +33,8 @@ import java.net.URL; import java.util.*; +import org.apache.zeppelin.conf.ZeppelinConfiguration; + /** * Load helium visualization & spell */ @@ -40,7 +42,6 @@ public class HeliumBundleFactory { Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class); private final String NODE_VERSION = "v6.9.1"; private final String NPM_VERSION = "3.10.8"; - private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/"; public static final String HELIUM_LOCAL_REPO = "helium-bundle"; public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js"; public static final String HELIUM_BUNDLE = "helium.bundle.js"; @@ -52,9 +53,11 @@ public class HeliumBundleFactory { private final FrontendPluginFactory frontEndPluginFactory; private final File workingDirectory; + private ZeppelinConfiguration conf; private File tabledataModulePath; private File visualizationModulePath; private File spellModulePath; + private String defaultNpmRegistryUrl; private Gson gson; private boolean nodeAndNpmInstalled = false; @@ -64,18 +67,23 @@ public class HeliumBundleFactory { ByteArrayOutputStream out = new ByteArrayOutputStream(); public HeliumBundleFactory( + ZeppelinConfiguration conf, File moduleDownloadPath, File tabledataModulePath, File visualizationModulePath, File spellModulePath) throws TaskRunnerException { - this(moduleDownloadPath); + this(conf, moduleDownloadPath); this.tabledataModulePath = tabledataModulePath; this.visualizationModulePath = visualizationModulePath; this.spellModulePath = spellModulePath; } - public HeliumBundleFactory(File moduleDownloadPath) throws TaskRunnerException { + public HeliumBundleFactory( + ZeppelinConfiguration conf, + File moduleDownloadPath) throws TaskRunnerException { this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO); + this.conf = conf; + this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry(); File installDirectory = workingDirectory; frontEndPluginFactory = new FrontendPluginFactory( @@ -412,7 +420,7 @@ private void npmCommand(String args) throws TaskRunnerException { private void npmCommand(String args, Map env) throws TaskRunnerException { installNodeAndNpm(); - NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL); + NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl); npm.execute(args, env); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java index 28cba4c193a..7b816e6b78a 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java @@ -30,10 +30,13 @@ import java.util.LinkedList; import java.util.List; +import org.apache.zeppelin.conf.ZeppelinConfiguration; + import static org.junit.Assert.*; public class HeliumBundleFactoryTest { private File tmpDir; + private ZeppelinConfiguration conf; private HeliumBundleFactory hbf; @Before @@ -46,7 +49,10 @@ public void setUp() throws InstallationException, TaskRunnerException { String resDir = new File(res.getFile()).getParent(); File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/"); - hbf = new HeliumBundleFactory(tmpDir, + conf = new ZeppelinConfiguration(); + + hbf = new HeliumBundleFactory(conf, + tmpDir, new File(moduleDir, "tabledata"), new File(moduleDir, "visualization"), new File(moduleDir, "spell"));