From 6f2b6f8e243548229ac26e18981308c89fc17d71 Mon Sep 17 00:00:00 2001 From: DuyHai DOAN Date: Fri, 26 Aug 2016 16:49:33 +0200 Subject: [PATCH 1/7] [ZEPPELIN-1376] Add proxy credentials information for dependencies repo --- .../dep/AbstractDependencyResolver.java | 3 +- .../org/apache/zeppelin/dep/Repository.java | 21 ++++++++++ .../zeppelin/rest/InterpreterRestApi.java | 2 +- .../app/interpreter/interpreter.controller.js | 7 +++- .../src/app/interpreter/interpreter.html | 4 +- .../repository-create/repository-dialog.html | 38 +++++++++++++++++++ .../interpreter/InterpreterFactory.java | 7 ++-- 7 files changed, 75 insertions(+), 7 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index 1e0844efaac..da3d0c02b4f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -72,7 +72,7 @@ public void addRepo(String id, String url, boolean snapshot) { } } - public void addRepo(String id, String url, boolean snapshot, Authentication auth) { + public void addRepo(String id, String url, boolean snapshot, Authentication auth, Proxy proxy) { synchronized (repos) { delRepo(id); RemoteRepository rr = new RemoteRepository(id, "default", url); @@ -81,6 +81,7 @@ public void addRepo(String id, String url, boolean snapshot, Authentication auth RepositoryPolicy.UPDATE_POLICY_DAILY, RepositoryPolicy.CHECKSUM_POLICY_WARN)); rr.setAuthentication(auth); + rr.setProxy(proxy); repos.add(rr); } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java index d2b00924607..34fe4f05478 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java @@ -16,7 +16,11 @@ */ package org.apache.zeppelin.dep; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; + /** * * @@ -27,6 +31,11 @@ public class Repository { private String url; private String username = null; private String password = null; + private String proxyProtocol = "HTTP"; + private String proxyHost = null; + private Integer proxyPort = null; + private String proxyLogin = null; + private String proxyPassword = null; public Repository(String id){ this.id = id; @@ -77,4 +86,16 @@ public Authentication getAuthentication() { } return auth; } + + public Proxy getProxy() { + if (isNotBlank(proxyHost) && proxyPort != null) { + if (isNotBlank(proxyLogin)) { + return new Proxy(proxyProtocol, proxyHost, proxyPort, + new Authentication(proxyLogin, proxyPassword)); + } else { + return new Proxy(proxyProtocol, proxyHost, proxyPort, null); + } + } + return null; + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 6025b52c7f1..34e8c427388 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -202,7 +202,7 @@ public Response addRepository(String message) { try { Repository request = gson.fromJson(message, Repository.class); interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(), - request.getAuthentication()); + request.getAuthentication(), request.getProxy()); logger.info("New repository {} added", request.getId()); } catch (Exception e) { logger.error("Exception in InterpreterRestApi while adding repository ", e); diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 97d4ea775f4..a57f2111623 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -508,7 +508,12 @@ url: '', snapshot: false, username: '', - password: '' + password: '', + proxyProtocol: 'HTTP', + proxyHost: '', + proxyPort: null, + proxyLogin: '', + proxyPassword: '' }; }; diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index f1c5e65982d..57b305ee67c 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -65,7 +65,9 @@

Repositories

popover-html-unsafe=" {{repo.url}}
- {{repo.authentication.username}}"> + {{repo.authentication.username}}
+ + {{repo.proxy.host}}"> {{repo.id}}  Add New Repository +
+

Proxy Settings (optional)

+
+
+
+ + + +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
From b2641932dc5b09d921bd902c4ebc9b04f9f0769b Mon Sep 17 00:00:00 2001 From: DuyHai DOAN Date: Sun, 28 Aug 2016 14:42:41 +0200 Subject: [PATCH 4/7] [ZEPPELIN-1376] Add unit test and fix impl for DependencyResolver to catch NPE --- .../apache/zeppelin/dep/DependencyResolver.java | 17 ++++++++++------- .../zeppelin/dep/DependencyResolverTest.java | 13 +++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index d6936e3001b..7ca4a4d173d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -33,6 +33,7 @@ import org.sonatype.aether.RepositoryException; import org.sonatype.aether.artifact.Artifact; import org.sonatype.aether.collection.CollectRequest; +import org.sonatype.aether.collection.DependencyCollectionException; import org.sonatype.aether.graph.Dependency; import org.sonatype.aether.graph.DependencyFilter; import org.sonatype.aether.repository.RemoteRepository; @@ -158,15 +159,14 @@ private List loadFromMvn(String artifact, Collection excludes) */ @Override public List getArtifactsWithDep(String dependency, - Collection excludes) throws RepositoryException { + Collection excludes) throws RepositoryException { Artifact artifact = new DefaultArtifact(dependency); DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); PatternExclusionsDependencyFilter exclusionFilter = - new PatternExclusionsDependencyFilter(excludes); + new PatternExclusionsDependencyFilter(excludes); CollectRequest collectRequest = new CollectRequest(); - final Dependency rootDependency = new Dependency(artifact, JavaScopes.COMPILE); - collectRequest.setRoot(rootDependency); + collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE)); synchronized (repos) { for (RemoteRepository repo : repos) { @@ -174,8 +174,11 @@ public List getArtifactsWithDep(String dependency, } } DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, - DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); - dependencyRequest.setRoot(new DefaultDependencyNode(rootDependency)); - return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); + DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); + try { + return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); + } catch (NullPointerException ex) { + throw new RepositoryException(String.format("Cannot fetch dependencies for %s", dependency)); + } } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index af2c7ff6963..876e8e7c88c 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import java.io.File; +import java.io.FileNotFoundException; import java.util.Collections; import org.apache.commons.io.FileUtils; @@ -36,6 +37,9 @@ public class DependencyResolverTest { private static File testCopyPath; private static File tmpDir; + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @BeforeClass public static void setUp() throws Exception { tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); @@ -90,4 +94,13 @@ public void testLoad() throws Exception { exception.expect(RepositoryException.class); resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); } + + @Test + public void should_throw_exception_if_dependency_not_found() throws Exception { + expectedException.expectMessage("Source 'one.two:1.0' does not exist"); + expectedException.expect(FileNotFoundException.class); + + resolver.load("one.two:1.0", testCopyPath); + } + } \ No newline at end of file From 9f76ef47326f48ebe7a447ff182b8d2af90501b9 Mon Sep 17 00:00:00 2001 From: DuyHai DOAN Date: Wed, 31 Aug 2016 20:29:47 +0200 Subject: [PATCH 5/7] [ZEPPELIN-1376] Do not repeat the same error popup multiple times --- zeppelin-web/src/app/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index f0f1d89cee5..603171f10a6 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -90,6 +90,7 @@ ngToastProvider.configure({ dismissButton: true, dismissOnClick: false, + combineDuplications: true, timeout: 6000 }); }); From 177fbd309fc78df7afa601cc1805d74087b5753f Mon Sep 17 00:00:00 2001 From: DuyHai DOAN Date: Thu, 8 Sep 2016 17:11:03 -0700 Subject: [PATCH 6/7] [ZEPPELIN-1376] Fixes JS bug to display error popup for other interpreters --- zeppelin-web/src/app/interpreter/interpreter.controller.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 5d22b7ad492..11894126fe2 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -108,10 +108,11 @@ var checkDownloadingDependencies = function() { var isDownloading = false; - for (var setting = 0; setting < $scope.interpreterSettings.length; setting++) { - var setting = $scope.interpreterSettings[setting]; + for (var index = 0; index < $scope.interpreterSettings.length; index++) { + var setting = $scope.interpreterSettings[index]; if (setting.status === 'DOWNLOADING_DEPENDENCIES') { isDownloading = true; + break; } else if (setting.status === 'ERROR') { ngToast.danger({content: 'Error setting properties for interpreter \'' + setting.group + '.' + setting.name + '\': ' + setting.errorReason, From b8d44e7d77601e4bac889a0e5c902200df9f2272 Mon Sep 17 00:00:00 2001 From: doanduyhai Date: Sat, 1 Oct 2016 22:24:21 +0200 Subject: [PATCH 7/7] [ZEPPELIN-1376] Improve error popup display --- .../src/app/configuration/configuration.controller.js | 1 + zeppelin-web/src/app/credential/credential.controller.js | 1 + zeppelin-web/src/app/home/home.controller.js | 6 ++++-- .../src/app/interpreter/interpreter.controller.js | 7 +++++-- zeppelin-web/src/app/jobmanager/jobmanager.controller.js | 5 +++-- zeppelin-web/src/app/notebook/notebook.controller.js | 9 +++++++-- .../apache/zeppelin/interpreter/InterpreterFactory.java | 2 ++ 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/zeppelin-web/src/app/configuration/configuration.controller.js b/zeppelin-web/src/app/configuration/configuration.controller.js index 611575e76a5..8960ca55f50 100644 --- a/zeppelin-web/src/app/configuration/configuration.controller.js +++ b/zeppelin-web/src/app/configuration/configuration.controller.js @@ -21,6 +21,7 @@ function ConfigurationCtrl($scope, $rootScope, $http, baseUrlSrv, ngToast) { $scope.configrations = []; $scope._ = _; + ngToast.dismiss(); var getConfigurations = function() { $http.get(baseUrlSrv.getRestApiBase() + '/configurations/all'). diff --git a/zeppelin-web/src/app/credential/credential.controller.js b/zeppelin-web/src/app/credential/credential.controller.js index bfb8bbacd63..7145299f708 100644 --- a/zeppelin-web/src/app/credential/credential.controller.js +++ b/zeppelin-web/src/app/credential/credential.controller.js @@ -20,6 +20,7 @@ function CredentialCtrl($scope, $rootScope, $http, baseUrlSrv, ngToast) { $scope._ = _; + ngToast.dismiss(); $scope.credentialInfo = []; $scope.showAddNewCredentialInfo = false; diff --git a/zeppelin-web/src/app/home/home.controller.js b/zeppelin-web/src/app/home/home.controller.js index 2c7407b7c0c..9063e33e482 100644 --- a/zeppelin-web/src/app/home/home.controller.js +++ b/zeppelin-web/src/app/home/home.controller.js @@ -21,10 +21,12 @@ 'notebookListDataFactory', 'websocketMsgSrv', '$rootScope', - 'arrayOrderingSrv' + 'arrayOrderingSrv', + 'ngToast' ]; - function HomeCtrl($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv) { + function HomeCtrl($scope, notebookListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv, ngToast) { + ngToast.dismiss(); var vm = this; vm.notes = notebookListDataFactory; vm.websocketMsgSrv = websocketMsgSrv; diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 11894126fe2..3fe355fe1b8 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -25,6 +25,7 @@ $scope.showAddNewSetting = false; $scope.showRepositoryInfo = false; $scope._ = _; + ngToast.dismiss(); $scope.openPermissions = function() { $scope.showInterpreterAuth = true; @@ -112,13 +113,15 @@ var setting = $scope.interpreterSettings[index]; if (setting.status === 'DOWNLOADING_DEPENDENCIES') { isDownloading = true; - break; - } else if (setting.status === 'ERROR') { + } + + if (setting.status === 'ERROR' || setting.errorReason) { ngToast.danger({content: 'Error setting properties for interpreter \'' + setting.group + '.' + setting.name + '\': ' + setting.errorReason, verticalPosition: 'top', dismissOnTimeout: false}); } } + if (isDownloading) { $timeout(function() { if ($route.current.$$route.originalPath === '/interpreter') { diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 2c6b67590c4..d28b374ec9d 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -16,9 +16,10 @@ angular.module('zeppelinWebApp').controller('JobmanagerCtrl', JobmanagerCtrl); - JobmanagerCtrl.$inject = ['$scope', 'websocketMsgSrv', '$interval']; + JobmanagerCtrl.$inject = ['$scope', 'websocketMsgSrv', '$interval', 'ngToast']; - function JobmanagerCtrl($scope, websocketMsgSrv, $interval) { + function JobmanagerCtrl($scope, websocketMsgSrv, $interval, ngToast) { + ngToast.dismiss(); $scope.filterValueToName = function(filterValue) { var index = _.findIndex($scope.ACTIVE_INTERPRETERS, {value: filterValue}); diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index da15768cf7b..f2aa67d2e86 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -26,11 +26,16 @@ 'websocketMsgSrv', 'baseUrlSrv', '$timeout', - 'saveAsService' + 'saveAsService', + 'ngToast' ]; function NotebookCtrl($scope, $route, $routeParams, $location, $rootScope, - $http, websocketMsgSrv, baseUrlSrv, $timeout, saveAsService) { + $http, websocketMsgSrv, baseUrlSrv, $timeout, saveAsService, + ngToast) { + + ngToast.dismiss(); + $scope.note = null; $scope.moment = moment; $scope.editorToggled = false; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 39bd060f48f..494361b304f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -433,6 +433,7 @@ public Map getEditorFromSettingByClassName(InterpreterSetting in private void loadInterpreterDependencies(final InterpreterSetting setting) { setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); + setting.setErrorReason(null); interpreterSettings.put(setting.getId(), setting); synchronized (interpreterSettings) { final Thread t = new Thread() { @@ -461,6 +462,7 @@ public void run() { } setting.setStatus(InterpreterSetting.Status.READY); + setting.setErrorReason(null); } catch (Exception e) { logger.error(String.format("Error while downloading repos for interpreter group : %s," + " go to interpreter setting page click on edit and save it again to make " +