Skip to content

Commit

Permalink
Use java to download external dependecies
Browse files Browse the repository at this point in the history
  • Loading branch information
Reamer committed Apr 3, 2024
1 parent 1613b6d commit 532fe2a
Show file tree
Hide file tree
Showing 19 changed files with 802 additions and 250 deletions.
10 changes: 6 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
<module>zeppelin-jupyter</module>
<module>zeppelin-plugins</module>
<module>zeppelin-distribution</module>
<module>zeppelin-test</module>
</modules>

<properties>
Expand All @@ -116,6 +117,7 @@
<!-- common library versions -->
<slf4j.version>1.7.35</slf4j.version>
<reload4j.version>1.2.25</reload4j.version>
<log4j2.version>2.23.1</log4j2.version>
<libthrift.version>0.13.0</libthrift.version>
<flexmark.all.version>0.62.2</flexmark.all.version>
<gson.version>2.8.9</gson.version>
Expand All @@ -125,13 +127,13 @@
<httpcomponents.core.version>4.4.1</httpcomponents.core.version>
<httpcomponents.client.version>4.5.13</httpcomponents.client.version>
<httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
<commons.compress.version>1.21</commons.compress.version>
<commons.lang3.version>3.12.0</commons.lang3.version>
<commons.compress.version>1.26.1</commons.compress.version>
<commons.lang3.version>3.14.0</commons.lang3.version>
<commons.text.version>1.10.0</commons.text.version>
<commons.configuration2.version>2.10.1</commons.configuration2.version>
<commons.exec.version>1.3</commons.exec.version>
<commons.codec.version>1.14</commons.codec.version>
<commons.io.version>2.7</commons.io.version>
<commons.codec.version>1.16.1</commons.codec.version>
<commons.io.version>2.15.1</commons.io.version>
<commons.collections.version>3.2.2</commons.collections.version>
<commons.cli.version>1.4</commons.cli.version>
<shiro.version>1.13.0</shiro.version>
Expand Down
8 changes: 4 additions & 4 deletions zeppelin-distribution/src/bin_license/LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,22 @@ The following components are provided under Apache License.
(Apache 2.0) Amazon Web Services SDK for Java v1.11.736 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.11.736/LICENSE.txt
(Apache 2.0) JavaEWAH v0.7.9 (https://github.com/lemire/javaewah) - https://github.com/lemire/javaewah/blob/master/LICENSE-2.0.txt
(Apache 2.0) Apache Commons Logging (commons-logging:commons-logging:1.1.1 - http://commons.apache.org/proper/commons-logging/)
(Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.5 - http://commons.apache.org/proper/commons-codec/)
(Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.16.1 - http://commons.apache.org/proper/commons-codec/)
(Apache 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/proper/commons-configuration/)
(Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - http://commons.apache.org/proper/commons-compress/)
(Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.1 - http://commons.apache.org/proper/commons-compress/)
(Apache 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.8.0 - http://commons.apache.org/configuration/)
(Apache 2.0) Apache Commons CLI (commons-cli:commons-cli:1.2 - http://commons.apache.org/cli/)
(Apache 2.0) Apache Commons Exec (commons-exec:commons-exec:1.3 - http://commons.apache.org/exec/)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpcore:4.3.3 - https://github.com/apache/httpclient)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpclient:4.3.6 - https://github.com/apache/httpclient)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpasyncclient:4.0.2 - https://github.com/apache/httpclient)
(Apache 2.0) Apache Commons Lang (org.apache.commons:commons-lang:2.6 - http://commons.apache.org/proper/commons-lang/)
(Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.10 - http://commons.apache.org/proper/commons-lang/)
(Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.14.0 - http://commons.apache.org/proper/commons-lang/)
(Apache 2.0) Apache Commons Math 3 (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
(Apache 2.0) Apache Commons Net (commons-net:commons-net:2.2 - http://commons.apache.org/proper/commons-net/)
(Apache 2.0) Apache Commons Pool2 (commons-exec:commons-pool2:2.3 - https://commons.apache.org/proper/commons-pool/)
(Apache 2.0) Apache Commons FileUpload (commons-fileupload:commons-fileupload:1.3.1 - http://commons.apache.org/fileupload/)
(Apache 2.0) Apache Commons IO (commons-io:commons-io:2.7 - http://commons.apache.org/io/)
(Apache 2.0) Apache Commons IO (commons-io:commons-io:2.15.1 - http://commons.apache.org/io/)
(Apache 2.0) Apache Commons VFS2 (org.apache.commons:commons-vfs2:2.0 - https://commons.apache.org/proper/commons-vfs/)
(Apache 2.0) Apache Jackrabbit webdav (org.apache.jackrabbit:jackrabbit-webdav:jar:1.5.2 - https://jackrabbit.apache.org/jcr/components/jackrabbit-webdav-library.html)
(Apache 2.0) Apache Jackrabbit JCR commons - http://jackrabbit.apache.org/jcr/components/jackrabbit-jcr-commons.html)
Expand Down
7 changes: 7 additions & 0 deletions zeppelin-interpreter-integration/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.interpreter.ExecutionContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
Expand All @@ -32,9 +33,7 @@
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.interpreter.ExecutionContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
Expand All @@ -36,7 +37,6 @@
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.interpreter.ExecutionContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
Expand All @@ -33,7 +34,6 @@
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -62,11 +62,9 @@ public class SparkSubmitIntegrationTest {

@BeforeAll
public static void setUp() throws IOException {
String sparkVersion = "3.4.1";
String hadoopVersion = "3";
LOGGER.info("Testing Spark Version: " + sparkVersion);
LOGGER.info("Testing Hadoop Version: " + hadoopVersion);
sparkHome = DownloadUtils.downloadSpark(sparkVersion, hadoopVersion);
LOGGER.info("Testing Spark Version: " + DownloadUtils.DEFAULT_SPARK_VERSION);
LOGGER.info("Testing Hadoop Version: " + DownloadUtils.DEFAULT_SPARK_HADOOP_VERSION);
sparkHome = DownloadUtils.downloadSpark();

hadoopCluster = new MiniHadoopCluster();
hadoopCluster.start();
Expand Down Expand Up @@ -102,7 +100,7 @@ void testLocalMode() throws InterpreterException, YarnException {
InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
InterpreterResult interpreterResult =
sparkSubmitInterpreter.interpret("--master local --class org.apache.spark.examples.SparkPi --deploy-mode client " +
sparkHome + "/examples/jars/spark-examples_2.12-3.4.1.jar", context);
sparkHome + "/examples/jars/spark-examples_2.12-" + DownloadUtils.DEFAULT_SPARK_VERSION + ".jar", context);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code(), interpreterResult.toString());

// no yarn application launched
Expand All @@ -126,7 +124,7 @@ void testYarnMode() throws InterpreterException, YarnException {
sparkSubmitInterpreter.interpret("--master yarn --deploy-mode cluster --class org.apache.spark.examples.SparkPi " +
"--conf spark.app.name=" + yarnAppName + " --conf spark.driver.memory=512m " +
"--conf spark.executor.memory=512m " +
sparkHome + "/examples/jars/spark-examples_2.12-3.4.1.jar", context);
sparkHome + "/examples/jars/spark-examples_2.12-" + DownloadUtils.DEFAULT_SPARK_VERSION + ".jar", context);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code(), interpreterResult.toString());

GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED));
Expand Down Expand Up @@ -159,7 +157,7 @@ public void run() {
sparkSubmitInterpreter.interpret("--master yarn --deploy-mode cluster --class org.apache.spark.examples.SparkPi " +
"--conf spark.app.name=" + yarnAppName + " --conf spark.driver.memory=512m " +
"--conf spark.executor.memory=512m " +
sparkHome + "/examples/jars/spark-examples_2.12-3.4.1.jar", context);
sparkHome + "/examples/jars/spark-examples_2.12-" + DownloadUtils.DEFAULT_SPARK_VERSION + ".jar", context);
assertEquals(InterpreterResult.Code.INCOMPLETE, interpreterResult.code(), interpreterResult.toString());
assertTrue(interpreterResult.toString().contains("Paragraph received a SIGTERM"), interpreterResult.toString());
} catch (InterpreterException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
package org.apache.zeppelin.integration;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.client.ClientConfig;
import org.apache.zeppelin.client.ExecuteResult;
import org.apache.zeppelin.client.websocket.SimpleMessageHandler;
import org.apache.zeppelin.client.Status;
import org.apache.zeppelin.client.ZSession;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.apache.zeppelin.interpreter.lifecycle.TimeoutLifecycleManager;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.rest.AbstractTestRestApi;
Expand Down Expand Up @@ -68,7 +68,7 @@ public static void setUp() throws Exception {
zConf.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_TIMEOUT_THRESHOLD.getVarName(), "10000");

notebook = TestUtils.getInstance(Notebook.class);
sparkHome = DownloadUtils.downloadSpark("3.4.1", "3");
sparkHome = DownloadUtils.downloadSpark();
flinkHome = DownloadUtils.downloadFlink("1.17.1", "2.12");
}

Expand Down Expand Up @@ -189,7 +189,7 @@ void testZSession_Spark() throws Exception {
assertEquals(Status.FINISHED, result.getStatus(), result.toString());
assertEquals(1, result.getResults().size());
assertEquals("TEXT", result.getResults().get(0).getType());
assertTrue(result.getResults().get(0).getData().contains("3.4.1"), result.getResults().get(0).getData());
assertTrue(result.getResults().get(0).getData().contains(DownloadUtils.DEFAULT_SPARK_VERSION), result.getResults().get(0).getData());
assertEquals(0, result.getJobUrls().size());

// pyspark
Expand Down Expand Up @@ -258,7 +258,7 @@ void testZSession_Spark_Submit() throws Exception {
assertEquals(Status.FINISHED, result.getStatus(), result.toString());
assertEquals(1, result.getResults().size());
assertEquals("TEXT", result.getResults().get(0).getType());
assertTrue(result.getResults().get(0).getData().contains("3.4.1"), result.getResults().get(0).getData());
assertTrue(result.getResults().get(0).getData().contains(DownloadUtils.DEFAULT_SPARK_VERSION), result.getResults().get(0).getData());
assertEquals(0, result.getJobUrls().size());

// pyspark
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
package org.apache.zeppelin.integration;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.rest.AbstractTestRestApi;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.zeppelin.integration;

import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.test.DownloadUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.Input;
Expand All @@ -29,19 +30,15 @@
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.integration.DownloadUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.utils.TestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ void should_throw_exception_if_dependency_not_found() throws Exception {
FileNotFoundException exception = assertThrows(FileNotFoundException.class, () -> {
resolver.load("one.two:1.0", testCopyPath);
});
assertEquals("Source 'one.two:1.0' does not exist", exception.getMessage());
assertEquals("File system element for parameter 'source' does not exist: 'one.two:1.0'",
exception.getMessage());
}

}
55 changes: 55 additions & 0 deletions zeppelin-test/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin</artifactId>
<version>0.12.0-SNAPSHOT</version>
</parent>
<artifactId>zeppelin-test</artifactId>
<name>Zeppelin Test</name>
<description>Zeppelin test code used in other modules</description>
<properties>
<progressbar.version>0.9.5</progressbar.version>
</properties>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${commons.compress.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>${progressbar.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.zeppelin.test;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Optional;

public class DownloadRequest {
private final URL url;
private final Optional<URL> alternativeUrl;
private final int retries;

public static final int DEFAULT_RETRIES = 3;

public DownloadRequest(String url, int retries) throws MalformedURLException {
this(url, null, retries);
}

public DownloadRequest(String url, String alternativeUrl) throws MalformedURLException {
this(url, alternativeUrl, DEFAULT_RETRIES);
}

public DownloadRequest(String url, String alternativeUrl, int retries)
throws MalformedURLException {
if (alternativeUrl != null) {
this.url = new URL(url);
this.alternativeUrl = Optional.of(new URL(alternativeUrl));
this.retries = retries;
} else {
this.url = new URL(url);
this.alternativeUrl = Optional.empty();
this.retries = retries;
}
}

public DownloadRequest(URL url, int retries) {
this(url, null, retries);
}

public DownloadRequest(URL url, URL alternativeUrl) {
this(url, alternativeUrl, DEFAULT_RETRIES);
}

public DownloadRequest(URL url, URL alternativeUrl, int retries) {
this.url = url;
this.alternativeUrl = Optional.of(alternativeUrl);
this.retries = retries;
}

public URL getUrl() {
return url;
}

public Optional<URL> getAlternativeUrl() {
return alternativeUrl;
}

public int getRetries() {
return retries;
}
}
Loading

0 comments on commit 532fe2a

Please sign in to comment.