From b773d8a775540509018e6be268980964538c4858 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Thu, 27 Jul 2017 21:55:22 +0100 Subject: [PATCH 1/3] [ZEPPELIN-2815] Improve interpreter dependencies logging --- .../main/java/org/apache/zeppelin/dep/Booter.java | 11 +++++++++-- .../apache/zeppelin/dep/DependencyResolver.java | 5 +++-- .../org/apache/zeppelin/dep/TransferListener.java | 14 ++++++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java index 5bc58edb5b3..635bf07cfd7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java @@ -18,7 +18,10 @@ package org.apache.zeppelin.dep; import org.apache.commons.lang.Validate; +import org.apache.log4j.lf5.LogLevel; import org.apache.maven.repository.internal.MavenRepositorySystemSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; import org.sonatype.aether.repository.LocalRepository; @@ -30,6 +33,8 @@ * Manage mvn repository. */ public class Booter { + private static Logger logger = LoggerFactory.getLogger(Booter.class); + public static RepositorySystem newRepositorySystem() { return RepositorySystemFactory.newRepositorySystem(); } @@ -43,8 +48,10 @@ public static RepositorySystemSession newRepositorySystemSession( LocalRepository localRepo = new LocalRepository(resolveLocalRepoPath(localRepoPath)); session.setLocalRepositoryManager(system.newLocalRepositoryManager(localRepo)); - // session.setTransferListener(new ConsoleTransferListener()); - // session.setRepositoryListener(new ConsoleRepositoryListener()); + if (logger.isDebugEnabled()) { + session.setTransferListener(new TransferListener()); + session.setRepositoryListener(new RepositoryListener()); + } // uncomment to generate dirty trees // session.setDependencyGraphTransformer( null ); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 889101fb6db..b5440838b8b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -177,8 +177,9 @@ public List getArtifactsWithDep(String dependency, DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); try { return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); - } catch (NullPointerException ex) { - throw new RepositoryException(String.format("Cannot fetch dependencies for %s", dependency)); + } catch (Exception ex) { + throw new RepositoryException( + String.format("Cannot fetch dependencies for %s", dependency), ex); } } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java index fd9029fa569..7b98578181f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.dep; -import java.io.PrintStream; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.Locale; @@ -34,8 +33,7 @@ * Simple listener that show deps downloading progress. */ public class TransferListener extends AbstractTransferListener { - Logger logger = LoggerFactory.getLogger(TransferListener.class); - private PrintStream out; + private Logger logger = LoggerFactory.getLogger(TransferListener.class); private Map downloads = new ConcurrentHashMap<>(); @@ -55,13 +53,13 @@ public void transferInitiated(TransferEvent event) { @Override public void transferProgressed(TransferEvent event) { TransferResource resource = event.getResource(); - downloads.put(resource, Long.valueOf(event.getTransferredBytes())); + downloads.put(resource, event.getTransferredBytes()); StringBuilder buffer = new StringBuilder(64); for (Map.Entry entry : downloads.entrySet()) { long total = entry.getKey().getContentLength(); - long complete = entry.getValue().longValue(); + long complete = entry.getValue(); buffer.append(getStatus(complete, total)).append(" "); } @@ -122,7 +120,7 @@ public void transferSucceeded(TransferEvent event) { @Override public void transferFailed(TransferEvent event) { transferCompleted(event); - event.getException().printStackTrace(out); + logger.warn(event.getException().getMessage()); } private void transferCompleted(TransferEvent event) { @@ -135,10 +133,10 @@ private void transferCompleted(TransferEvent event) { @Override public void transferCorrupted(TransferEvent event) { - event.getException().printStackTrace(out); + logger.error(event.getException().getMessage()); } - protected long toKB(long bytes) { + private long toKB(long bytes) { return (bytes + 1023) / 1024; } From d381f3b31ec9a6c95538324ef6d9383ea2acaff1 Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Mon, 7 Aug 2017 07:41:37 +0100 Subject: [PATCH 2/3] [ZEPPELIN-2815] Improve interpreter dependencies logging --- .../src/main/java/org/apache/zeppelin/dep/Booter.java | 2 -- .../java/org/apache/zeppelin/dep/DependencyResolver.java | 9 +++------ .../java/org/apache/zeppelin/dep/TransferListener.java | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java index 635bf07cfd7..6339a4f02b8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.dep; import org.apache.commons.lang.Validate; -import org.apache.log4j.lf5.LogLevel; import org.apache.maven.repository.internal.MavenRepositorySystemSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +51,6 @@ public static RepositorySystemSession newRepositorySystemSession( session.setTransferListener(new TransferListener()); session.setRepositoryListener(new RepositoryListener()); } - // uncomment to generate dirty trees // session.setDependencyGraphTransformer( null ); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index b5440838b8b..c3ecdeedc13 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -33,25 +32,23 @@ import org.sonatype.aether.RepositoryException; import org.sonatype.aether.artifact.Artifact; import org.sonatype.aether.collection.CollectRequest; -import org.sonatype.aether.collection.DependencyCollectionException; import org.sonatype.aether.graph.Dependency; import org.sonatype.aether.graph.DependencyFilter; import org.sonatype.aether.repository.RemoteRepository; import org.sonatype.aether.resolution.ArtifactResult; import org.sonatype.aether.resolution.DependencyRequest; +import org.sonatype.aether.resolution.DependencyResolutionException; import org.sonatype.aether.util.artifact.DefaultArtifact; import org.sonatype.aether.util.artifact.JavaScopes; import org.sonatype.aether.util.filter.DependencyFilterUtils; import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; -import org.sonatype.aether.util.graph.DefaultDependencyNode; - /** * Deps resolver. * Add new dependencies from mvn repo (at runtime) to Zeppelin. */ public class DependencyResolver extends AbstractDependencyResolver { - Logger logger = LoggerFactory.getLogger(DependencyResolver.class); + private Logger logger = LoggerFactory.getLogger(DependencyResolver.class); private final String[] exclusions = new String[] {"org.apache.zeppelin:zeppelin-zengine", "org.apache.zeppelin:zeppelin-interpreter", @@ -177,7 +174,7 @@ public List getArtifactsWithDep(String dependency, DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); try { return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); - } catch (Exception ex) { + } catch (NullPointerException | DependencyResolutionException ex) { throw new RepositoryException( String.format("Cannot fetch dependencies for %s", dependency), ex); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java index 7b98578181f..57b280e65ba 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java @@ -133,7 +133,7 @@ private void transferCompleted(TransferEvent event) { @Override public void transferCorrupted(TransferEvent event) { - logger.error(event.getException().getMessage()); + logger.error("Corrupted transfer", event.getException()); } private long toKB(long bytes) { From ea353fa2c16c38f8408b922bb61a9dfa269afdea Mon Sep 17 00:00:00 2001 From: Nelson Costa Date: Tue, 8 Aug 2017 08:11:17 +0100 Subject: [PATCH 3/3] [ZEPPELIN-2815] Improve interpreter dependencies logging --- .../src/main/java/org/apache/zeppelin/dep/TransferListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java index 57b280e65ba..7f25e3bafae 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java @@ -120,7 +120,7 @@ public void transferSucceeded(TransferEvent event) { @Override public void transferFailed(TransferEvent event) { transferCompleted(event); - logger.warn(event.getException().getMessage()); + logger.warn("Unsuccessful transfer", event.getException()); } private void transferCompleted(TransferEvent event) {