From 88b85c864ba82d0935a9834015e13358aaca46ec Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Thu, 26 Nov 2015 07:41:19 -0500 Subject: [PATCH 01/17] Deprecated Authenticator.newSaslNegotiator() and added new method to replace. --- .../server/auth/AllowAllAuthenticator.java | 10 +++++++ .../gremlin/server/auth/Authenticator.java | 27 ++++++++++++++++--- .../server/auth/SimpleAuthenticator.java | 9 +++++++ .../handler/SaslAuthenticationHandler.java | 22 ++++++++++++++- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/AllowAllAuthenticator.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/AllowAllAuthenticator.java index 8f062fa90f2..bce2dd6956c 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/AllowAllAuthenticator.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/AllowAllAuthenticator.java @@ -18,6 +18,7 @@ */ package org.apache.tinkerpop.gremlin.server.auth; +import java.net.InetAddress; import java.util.Map; /** @@ -41,7 +42,16 @@ public AuthenticatedUser authenticate(final Map credentials) thr return AuthenticatedUser.ANONYMOUS_USER; } + /** + * @deprecated As of release 3.1.1-incubating, replaced by {@link #newSaslNegotiator(InetAddress)}. + * @see TINKERPOP3-995 + */ + @Override + @Deprecated public SaslNegotiator newSaslNegotiator() { + // While this method is deprecated, it remains here to ensure backward compatibility with the old method. In + // this way the integration tests can continue to execute here + // todo: remove this method on a future version and implement the new one return AUTHENTICATOR_INSTANCE; } diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/Authenticator.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/Authenticator.java index 873ae095ffd..f282ad9df4f 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/Authenticator.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/Authenticator.java @@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; import org.apache.tinkerpop.gremlin.server.Channelizer; +import java.net.InetAddress; import java.util.Map; /** @@ -45,12 +46,32 @@ public interface Authenticator { public void setup(final Map config); /** - * Provide a SASL handler to perform authentication for an single connection. SASL - * is a stateful protocol, so a new instance must be used for each authentication - * attempt.) + * Provide a SASL handler to perform authentication for an single connection. SASL is a stateful protocol, so a + * new instance must be used for each authentication attempt.) + * + * @deprecated As of release 3.1.1-incubating, replaced by {@link #newSaslNegotiator(InetAddress)}. + * @see TINKERPOP3-995 */ + @Deprecated public SaslNegotiator newSaslNegotiator(); + /** + * Provide a SASL handler to perform authentication for an single connection. SASL is a stateful protocol, so + * a new instance must be used for each authentication attempt. + * + * As of 3.1.1, this method by default calls the {@link #newSaslNegotiator()} method so as not to introduce a + * breaking change. Implementers should move their code from {@link #newSaslNegotiator()} to this method as + * this is the method now called by Gremlin Server during authentication. For full backwards compatibility, + * it makes sense to call this method from {@link #newSaslNegotiator()} passing {@code null} for the + * {@code remoteAddress} parameter. + * + * @param remoteAddress the IP address of the client to authenticate to authenticate or null if an internal + * client (one not connected over the remote transport). + */ + public default SaslNegotiator newSaslNegotiator(final InetAddress remoteAddress) { + return newSaslNegotiator(); + } + /** * A "standard" authentication implementation that can be used more generically without SASL support. This * implementation is used when a particular {@link Channelizer} doesn't support SASL directly (like basic diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java index 769234375c6..4287cfae1e2 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.InetAddress; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; @@ -104,8 +105,16 @@ public void setup(final Map config) { logger.info("CredentialGraph initialized at {}", credentialStore); } + /** + * @deprecated As of release 3.1.1-incubating, replaced by {@link #newSaslNegotiator(InetAddress)}. + * @see TINKERPOP3-995 + */ @Override + @Deprecated public SaslNegotiator newSaslNegotiator() { + // While this method is deprecated, it remains here to ensure backward compatibility with the old method. In + // this way the integration tests can continue to execute here + // todo: remove this method on a future version and implement the new one return new PlainTextSaslAuthenticator(); } diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/SaslAuthenticationHandler.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/SaslAuthenticationHandler.java index 93cb168e588..de7f624a7c5 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/SaslAuthenticationHandler.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/SaslAuthenticationHandler.java @@ -18,10 +18,15 @@ */ package org.apache.tinkerpop.gremlin.server.handler; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.util.Attribute; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; import java.util.Base64; import org.apache.tinkerpop.gremlin.driver.Tokens; import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; @@ -61,7 +66,7 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) throw final Attribute request = ctx.attr(StateKey.REQUEST_MESSAGE); if (negotiator.get() == null) { // First time through so save the request and send an AUTHENTICATE challenge with no data - negotiator.set(authenticator.newSaslNegotiator()); + negotiator.set(authenticator.newSaslNegotiator(getRemoteInetAddress(ctx))); request.set(requestMessage); final ResponseMessage authenticate = ResponseMessage.build(requestMessage) .code(ResponseStatusCode.AUTHENTICATE).create(); @@ -121,4 +126,19 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) throw ctx.close(); } } + + private InetAddress getRemoteInetAddress(ChannelHandlerContext ctx) + { + Channel channel = ctx.channel(); + + if (null == channel) + return null; + + SocketAddress genericSocketAddr = channel.remoteAddress(); + + if (null == genericSocketAddr || !(genericSocketAddr instanceof InetSocketAddress)) + return null; + + return ((InetSocketAddress)genericSocketAddr).getAddress(); + } } From aec1bfff9ead41a5c7ebad542b872bfb00adb43d Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Tue, 1 Dec 2015 06:09:38 -0500 Subject: [PATCH 02/17] Update upgrade docs on Authenticator changes. --- docs/src/upgrade/release-3.1.x-incubating.asciidoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/src/upgrade/release-3.1.x-incubating.asciidoc b/docs/src/upgrade/release-3.1.x-incubating.asciidoc index 54ccf04d9f5..5cccb9439e4 100644 --- a/docs/src/upgrade/release-3.1.x-incubating.asciidoc +++ b/docs/src/upgrade/release-3.1.x-incubating.asciidoc @@ -60,6 +60,20 @@ TinkerGraph instances. See: link:https://issues.apache.org/jira/browse/TINKERPOP3-886[TINKERPOP3-886] +Authenticator Method Deprecation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For users who have a custom `Authenticator` implementation for Gremlin Server, there will be a new method present: + +[source,java] +public default SaslNegotiator newSaslNegotiator(final InetAddress remoteAddress) + +Implementation of this method is now preferred over the old method with the same name that has no arguments. The old +method has been deprecated. This is a non-breaking change as the new method has a default implementation that simply +calls the old deprecated method. In this way, existing `Authenticator` implementations will still work. + +See: link:https://issues.apache.org/jira/browse/TINKERPOP3-995[TINKERPOP3-995] + Upgrading for Providers ~~~~~~~~~~~~~~~~~~~~~~~ From 6f8c580184dd3e02b94bed4764d3948ff785a249 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Tue, 1 Dec 2015 07:07:33 -0500 Subject: [PATCH 03/17] Updated changelog. --- CHANGELOG.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1084fc5036d..bb969c6d7ac 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,9 +26,10 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/ TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* Deprecate `credentialsDbLocation` from `SimpleAuthenticator` in Gremlin Server. +* Deprecated `credentialsDbLocation` from `SimpleAuthenticator` in Gremlin Server. * `TinkerGraph` has "native" serialization in GraphSON, which enables it to be a return value from Gremlin Server. * Improved the ability to embed Gremlin Server by providing a way to get the `ServerGremlinExecutor` and improve reusability of `AbstractEvalOpProcessor` and related classes. +* Added `Authenticator.newSaslNegotiator(InetAddress)` and deprecated the zero-arg version of that method. * `ProfileStep` is now available off of `Traversal` via `profile()`. To be consistent with `Traversal.explain()`. * If no comparator is provided to `order()`, `Order.incr` is assumed (previously, an exception occurred). * Fixed various Gremlin-Groovy tests that assumed `toString()`-able ids. From b503e41552e533bc4420a968ba591faacd1c6e27 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 2 Dec 2015 12:44:36 -0500 Subject: [PATCH 04/17] Adjusted ScriptEngines to "merge" bindings. Bindings from plugins were being pushed into the ScriptEngine context but were not being merged into the bindings used on eval. They had to be merged together. In doing this, I also found a bug where plugins could conflict with one another (i.e. the first plugin would install, then the second would come along an blow out what the previous one had done) - fixed that. Added test cases to cover these scenarios. --- ...remlinGroovyScriptEngineOverGraphTest.java | 7 +- .../gremlin/groovy/engine/ScriptEngines.java | 24 ++++- .../jsr223/GremlinGroovyScriptEngine.java | 19 +++- .../jsr223/ScriptEnginePluginAcceptor.java | 26 +++-- .../groovy/engine/ScriptEnginesTest.java | 99 +++++++++++++++++++ .../jsr223/GremlinGroovyScriptEngineTest.java | 35 ++++++- .../io/script/ScriptRecordReader.java | 3 +- .../io/script/ScriptRecordWriter.java | 3 +- 8 files changed, 194 insertions(+), 22 deletions(-) diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineOverGraphTest.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineOverGraphTest.java index 5e93d1c99ca..17e1cf0aa57 100644 --- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineOverGraphTest.java +++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineOverGraphTest.java @@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.AbstractGremlinTest; import org.apache.tinkerpop.gremlin.FeatureRequirementSet; import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.NoImportCustomizerProvider; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; @@ -72,7 +73,7 @@ public void shouldDoSomeGremlin() throws Exception { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) public void shouldLoadImports() throws Exception { - final ScriptEngine engineNoImports = new GremlinGroovyScriptEngine(NoImportCustomizerProvider.INSTANCE); + final ScriptEngine engineNoImports = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); try { engineNoImports.eval("Vertex.class.getName()"); fail("Should have thrown an exception because no imports were supplied"); @@ -80,7 +81,7 @@ public void shouldLoadImports() throws Exception { assertTrue(se instanceof ScriptException); } - final ScriptEngine engineWithImports = new GremlinGroovyScriptEngine(new DefaultImportCustomizerProvider()); + final ScriptEngine engineWithImports = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) new DefaultImportCustomizerProvider()); engineWithImports.put("g", g); assertEquals(Vertex.class.getName(), engineWithImports.eval("Vertex.class.getName()")); assertEquals(2l, engineWithImports.eval("g.V().has('age',gt(30)).count().next()")); @@ -93,7 +94,7 @@ public void shouldLoadImports() throws Exception { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) public void shouldLoadStandardImportsAndThenAddToThem() throws Exception { - final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(new DefaultImportCustomizerProvider()); + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) new DefaultImportCustomizerProvider()); engine.put("g", g); assertEquals(Vertex.class.getName(), engine.eval("Vertex.class.getName()")); assertEquals(2l, engine.eval("g.V().has('age',gt(30)).count().next()")); diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java index 70a47a6dc9b..a03acee2b29 100644 --- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java +++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java @@ -31,6 +31,7 @@ import javax.script.Bindings; import javax.script.Compilable; import javax.script.CompiledScript; +import javax.script.ScriptContext; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; @@ -93,7 +94,11 @@ public Object eval(final String script, final Bindings bindings, final String la throw new IllegalArgumentException(String.format("Language [%s] not supported", language)); awaitControlOp(); - return scriptEngines.get(language).eval(script, bindings); + + final ScriptEngine engine = scriptEngines.get(language); + final Bindings all = mergeBindings(bindings, engine); + + return engine.eval(script, all); } /** @@ -105,7 +110,11 @@ public Object eval(final Reader reader, final Bindings bindings, final String la throw new IllegalArgumentException("Language [%s] not supported"); awaitControlOp(); - return scriptEngines.get(language).eval(reader, bindings); + + final ScriptEngine engine = scriptEngines.get(language); + final Bindings all = mergeBindings(bindings, engine); + + return engine.eval(reader, all); } /** @@ -389,4 +398,15 @@ private static synchronized Optional createScriptEngine(final Stri } } + /** + * Takes the bindings from a request for eval and merges them with the {@code ENGINE_SCOPE} bindings. + */ + private static Bindings mergeBindings(final Bindings bindings, final ScriptEngine engine) { + // plugins place "globals" here - see ScriptEnginePluginAcceptor + final Bindings all = engine.getBindings(ScriptContext.ENGINE_SCOPE); + + // merge the globals with the incoming bindings where local bindings "win" + all.putAll(bindings); + return all; + } } diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java index 2feaf71d2a6..72e38c31520 100644 --- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java +++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java @@ -286,7 +286,7 @@ public synchronized void addImports(final Set importStatements) { // use the EmptyImportCustomizer because it doesn't come with static initializers containing // existing imports. importCustomizerProvider = new EmptyImportCustomizerProvider(importCustomizerProvider, imports, staticImports); - reset(); + internalReset(); } /** @@ -315,6 +315,19 @@ public void close() throws Exception { */ @Override public void reset() { + internalReset(); + + loadedPlugins.clear(); + + getContext().getBindings(ScriptContext.ENGINE_SCOPE).clear(); + } + + /** + * Resets the {@code ScriptEngine} but does not clear the loaded plugins or bindings. Typically called by + * {@link DependencyManager} methods that need to just force the classloader to be recreated and script caches + * cleared. + */ + private void internalReset() { createClassLoader(); // must clear the local cache here because the the classloader has been reset. therefore, classes previously @@ -322,12 +335,8 @@ public void reset() { classMap.clear(); globalClosures.clear(); - loadedPlugins.clear(); - final Set toReuse = new HashSet<>(artifactsToUse); toReuse.forEach(this::use); - - getContext().getBindings(ScriptContext.ENGINE_SCOPE).clear(); } /** diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/ScriptEnginePluginAcceptor.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/ScriptEnginePluginAcceptor.java index 00817b18dff..1a72864c281 100644 --- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/ScriptEnginePluginAcceptor.java +++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/ScriptEnginePluginAcceptor.java @@ -29,6 +29,9 @@ import java.util.Set; /** + * A {@link PluginAcceptor} implementation for bare {@code ScriptEngine} implementations allowing plugins to + * interact with them on initialization. + * * @author Stephen Mallette (http://stephen.genoprime.com) */ public class ScriptEnginePluginAcceptor implements PluginAcceptor { @@ -38,19 +41,27 @@ public ScriptEnginePluginAcceptor(final ScriptEngine scriptEngine) { this.scriptEngine = scriptEngine; } + /** + * Adds global bindings to the {@code ScriptEngine} that will be applied to every evaluated script. + */ @Override public void addBinding(final String key, final Object val) { - scriptEngine.getContext().setAttribute(key, val, ScriptContext.GLOBAL_SCOPE); + // added to the engine scope because the plugin will be applied to each scriptengine independently anyway + scriptEngine.getContext().setAttribute(key, val, ScriptContext.ENGINE_SCOPE); } + /** + * Gets the global bindings that will be applied to every evaluated script. + */ @Override public Map getBindings() { - return scriptEngine.getBindings(ScriptContext.GLOBAL_SCOPE); + // as these "global" bindings were added to engine scope they should be pulled from the same place + return scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE); } /** - * If the ScriptEngine implements the DependencyManager interface it will try to import the specified - * import statements. + * If the {@code ScriptEngine} implements the {@link DependencyManager} interface it will try to import the + * specified import statements. */ @Override public void addImports(final Set importStatements) { @@ -59,14 +70,17 @@ public void addImports(final Set importStatements) { } /** - * Evaluate a script in the ScriptEngine. Typically eval() should be called after imports as ScriptEngine - * resets may occur during import. + * Evaluate a script in the {@code ScriptEngine}. Typically {@code eval()} should be called after imports as + * {@code ScriptEngine} resets may occur during import. */ @Override public Object eval(final String script) throws ScriptException { return this.scriptEngine.eval(script); } + /** + * Defines the environment settings for the {@link GremlinPlugin}. + */ @Override public Map environment() { final Map env = new HashMap<>(); diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java index 39e14fd96c8..940a5304b7c 100644 --- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java +++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEnginesTest.java @@ -18,21 +18,120 @@ */ package org.apache.tinkerpop.gremlin.groovy.engine; +import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin; +import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException; +import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; +import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException; import org.junit.Test; +import javax.script.Bindings; import javax.script.SimpleBindings; +import java.awt.*; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** * @author Stephen Mallette (http://stephen.genoprime.com) */ public class ScriptEnginesTest { + @Test + public void shouldMergeBindingsFromLocalAndGlobal() throws Exception { + final ScriptEngines engines = new ScriptEngines(se -> {}); + engines.reload("gremlin-groovy", Collections.emptySet(), + Collections.emptySet(), Collections.emptyMap()); + + engines.loadPlugins(Arrays.asList(new GremlinPlugin() { + @Override + public String getName() { + return "mock"; + } + + @Override + public void pluginTo(final PluginAcceptor pluginAcceptor) throws IllegalEnvironmentException, PluginInitializationException { + pluginAcceptor.addBinding("y", "here"); + } + })); + + final Bindings localBindings = new SimpleBindings(); + localBindings.put("x", "there"); + + assertEquals("herethere", engines.eval("y+x", localBindings, "gremlin-groovy")); + } + + @Test + public void shouldMergeBindingsFromLocalAndGlobalWithMultiplePlugins() throws Exception { + final ScriptEngines engines = new ScriptEngines(se -> {}); + engines.reload("gremlin-groovy", Collections.emptySet(), + Collections.emptySet(), Collections.emptyMap()); + + engines.loadPlugins(Arrays.asList(new GremlinPlugin() { + @Override + public String getName() { + return "mock1"; + } + + @Override + public void pluginTo(final PluginAcceptor pluginAcceptor) throws IllegalEnvironmentException, PluginInitializationException { + pluginAcceptor.addBinding("y", "here"); + } + })); + + final Bindings localBindings = new SimpleBindings(); + localBindings.put("x", "there"); + + assertEquals("herethere", engines.eval("y+x", localBindings, "gremlin-groovy")); + + engines.loadPlugins(Arrays.asList(new GremlinPlugin() { + @Override + public String getName() { + return "mock2"; + } + + @Override + public void pluginTo(final PluginAcceptor pluginAcceptor) throws IllegalEnvironmentException, PluginInitializationException { + pluginAcceptor.addBinding("z", "where"); + pluginAcceptor.addImports(new HashSet<>(Arrays.asList("import java.awt.Color"))); + } + })); + + assertEquals("heretherewhere", engines.eval("y+x+z", localBindings, "gremlin-groovy")); + assertEquals(Color.RED, engines.eval("Color.RED", localBindings, "gremlin-groovy")); + + } + + @Test + public void shouldMergeBindingsWhereLocalOverridesGlobal() throws Exception { + final ScriptEngines engines = new ScriptEngines(se -> {}); + engines.reload("gremlin-groovy", Collections.emptySet(), + Collections.emptySet(), Collections.emptyMap()); + + engines.loadPlugins(Arrays.asList(new GremlinPlugin() { + @Override + public String getName() { + return "mock"; + } + + @Override + public void pluginTo(final PluginAcceptor pluginAcceptor) throws IllegalEnvironmentException, PluginInitializationException { + pluginAcceptor.addBinding("y", "here"); + } + })); + + // the "y" below should override the global variable setting. + final Bindings localBindings = new SimpleBindings(); + localBindings.put("y", "there"); + localBindings.put("z", "where"); + + assertEquals("therewhere", engines.eval("y+z", localBindings, "gremlin-groovy")); + } + @Test public void shouldFailUntilImportExecutes() throws Exception { final ScriptEngines engines = new ScriptEngines(se -> {}); diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java index 9e74686a58d..7480e01b729 100644 --- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java +++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineTest.java @@ -19,6 +19,7 @@ package org.apache.tinkerpop.gremlin.groovy.jsr223; import groovy.lang.Closure; +import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.NoImportCustomizerProvider; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; @@ -29,6 +30,7 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; import javax.script.SimpleBindings; +import java.awt.*; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -98,7 +100,7 @@ public void shouldEvalFailingAssert() throws Exception { @Test public void shouldLoadImportsViaDependencyManagerInterface() throws Exception { - final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(NoImportCustomizerProvider.INSTANCE); + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); try { engine.eval("Vertex.class.getName()"); fail("Should have thrown an exception because no imports were supplied"); @@ -112,7 +114,7 @@ public void shouldLoadImportsViaDependencyManagerInterface() throws Exception { @Test public void shouldLoadImportsViaDependencyManagerInterfaceAdditively() throws Exception { - final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(NoImportCustomizerProvider.INSTANCE); + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); try { engine.eval("Vertex.class.getName()"); fail("Should have thrown an exception because no imports were supplied"); @@ -144,7 +146,7 @@ public void shouldLoadImportsViaDependencyManagerInterfaceAdditively() throws Ex @Test public void shouldLoadImportsViaDependencyManagerFromDependencyGatheredByUse() throws Exception { - final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(NoImportCustomizerProvider.INSTANCE); + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); try { engine.eval("org.apache.commons.math3.util.FastMath.abs(-1235)"); fail("Should have thrown an exception because no imports were supplied"); @@ -159,7 +161,7 @@ public void shouldLoadImportsViaDependencyManagerFromDependencyGatheredByUse() t @Test public void shouldAllowsUseToBeExecutedAfterImport() throws Exception { - final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(NoImportCustomizerProvider.INSTANCE); + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); try { engine.eval("org.apache.commons.math3.util.FastMath.abs(-1235)"); fail("Should have thrown an exception because no imports were supplied"); @@ -172,6 +174,31 @@ public void shouldAllowsUseToBeExecutedAfterImport() throws Exception { assertEquals(1235, engine.eval("org.apache.commons.math3.util.FastMath.abs(-1235)")); } + @Test + public void shouldAllowsMultipleImports() throws Exception { + final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) NoImportCustomizerProvider.INSTANCE); + try { + engine.eval("Color.RED"); + fail("Should have thrown an exception because no imports were supplied"); + } catch (Exception se) { + assertTrue(se instanceof ScriptException); + } + + try { + engine.eval("SystemColor.ACTIVE_CAPTION"); + fail("Should have thrown an exception because no imports were supplied"); + } catch (Exception se) { + assertTrue(se instanceof ScriptException); + } + + engine.addImports(new HashSet<>(Arrays.asList("import java.awt.Color"))); + assertEquals(Color.RED, engine.eval("Color.RED")); + + engine.addImports(new HashSet<>(Arrays.asList("import java.awt.SystemColor"))); + assertEquals(Color.RED, engine.eval("Color.RED")); + assertEquals(SystemColor.ACTIVE_CAPTION, engine.eval("SystemColor.ACTIVE_CAPTION")); + } + @Test public void shouldClearEngineScopeOnReset() throws Exception { final GremlinGroovyScriptEngine engine = new GremlinGroovyScriptEngine(); diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java index e56eb1a59cc..4cc1602b08c 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReader.java @@ -26,6 +26,7 @@ import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; +import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; @@ -64,7 +65,7 @@ public ScriptRecordReader() { public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { this.lineRecordReader.initialize(genericSplit, context); final Configuration configuration = context.getConfiguration(); - this.engine = new GremlinGroovyScriptEngine(new DefaultImportCustomizerProvider()); + this.engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) new DefaultImportCustomizerProvider()); //this.engine = ScriptEngineCache.get(configuration.get(SCRIPT_ENGINE, ScriptEngineCache.DEFAULT_SCRIPT_ENGINE)); final FileSystem fs = FileSystem.get(configuration); try { diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java index 5cf61701bd9..d445fcdade8 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java @@ -24,6 +24,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider; import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; @@ -61,7 +62,7 @@ public final class ScriptRecordWriter extends RecordWriter Date: Fri, 4 Dec 2015 09:08:00 -0500 Subject: [PATCH 05/17] Improved the ability to control logging for maven builds. Added a "silent" configuration for the "ci" profile so that travis will be happy. --- CHANGELOG.asciidoc | 1 + docs/src/dev/developer/contributing.asciidoc | 13 ++++- giraph-gremlin/pom.xml | 18 ++----- .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 23 +++++++++ gremlin-console/pom.xml | 6 +++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ gremlin-core/pom.xml | 6 +++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ ...=> GremlinServerAuthOldIntegrateTest.java} | 2 +- .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ gremlin-test/pom.xml | 6 +++ .../gremlin/AbstractGremlinTest.java | 6 +++ .../test/resources/log4j-silent.properties | 21 +++++++++ .../src/test/resources/log4j-test.properties | 24 ++++++++++ hadoop-gremlin/pom.xml | 18 ++----- ...est.java => HadoopGremlinPluginCheck.java} | 5 +- .../groovy/plugin/HadoopPluginSuite.java | 2 +- .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ neo4j-gremlin/pom.xml | 6 +++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ pom.xml | 47 ++++++++++--------- spark-gremlin/pom.xml | 18 ++----- .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 25 ++++++++++ .../test/resources/log4j-silent.properties | 18 +++++++ .../src/test/resources/log4j-test.properties | 21 +++++++++ 36 files changed, 526 insertions(+), 69 deletions(-) create mode 100644 giraph-gremlin/src/test/resources/log4j-silent.properties create mode 100644 giraph-gremlin/src/test/resources/log4j-test.properties create mode 100644 gremlin-console/src/test/resources/log4j-silent.properties create mode 100644 gremlin-console/src/test/resources/log4j-test.properties create mode 100644 gremlin-core/src/test/resources/log4j-silent.properties create mode 100644 gremlin-core/src/test/resources/log4j-test.properties create mode 100644 gremlin-driver/src/test/resources/log4j-silent.properties create mode 100644 gremlin-driver/src/test/resources/log4j-test.properties create mode 100644 gremlin-groovy/src/test/resources/log4j-silent.properties create mode 100644 gremlin-groovy/src/test/resources/log4j-test.properties rename gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/{GremlinServerAuthIntegrateOldTest.java => GremlinServerAuthOldIntegrateTest.java} (99%) create mode 100644 gremlin-server/src/test/resources/log4j-silent.properties create mode 100644 gremlin-server/src/test/resources/log4j-test.properties create mode 100644 gremlin-test/src/test/resources/log4j-silent.properties create mode 100644 gremlin-test/src/test/resources/log4j-test.properties rename hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/{HadoopGremlinPluginTest.java => HadoopGremlinPluginCheck.java} (97%) create mode 100644 hadoop-gremlin/src/test/resources/log4j-silent.properties create mode 100644 hadoop-gremlin/src/test/resources/log4j-test.properties create mode 100644 neo4j-gremlin/src/test/resources/log4j-silent.properties create mode 100644 neo4j-gremlin/src/test/resources/log4j-test.properties create mode 100644 spark-gremlin/src/test/resources/log4j-silent.properties create mode 100644 spark-gremlin/src/test/resources/log4j-test.properties create mode 100644 tinkergraph-gremlin/src/test/resources/log4j-silent.properties create mode 100644 tinkergraph-gremlin/src/test/resources/log4j-test.properties diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 717aa71e3a9..1fa03d530f2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/ TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Improved logging control during builds with Maven. * `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated. * Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark. * Added `AbstractSparkTest` which handles closing `SparkContext` instances between tests now that we support persisted contexts. diff --git a/docs/src/dev/developer/contributing.asciidoc b/docs/src/dev/developer/contributing.asciidoc index 70c78e2807f..959f4ef9942 100644 --- a/docs/src/dev/developer/contributing.asciidoc +++ b/docs/src/dev/developer/contributing.asciidoc @@ -401,4 +401,15 @@ the existing `` entries, paying special attention to the pathing of t uploaded to the server and should preserve the directory structure in git as referenced in ``. Please see the <> section for more information on how to generate the -documentation. \ No newline at end of file +documentation. + +[[logging]] +Logging +~~~~~~~ + +TinkerPop uses SLF4j for logging and typically leans back on Log4j as the implementation. Configuring log outputs +for debugging purposes within tests can be altered by editing the `log4j-test.properties` file in each module's test +resources. That file gets copied to the `target/test-classes` on build and surefire and failsafe plugins in maven +are then configured to point at that area of the file system for those configuration files. The properties files +can be edited to fine tune control of the log output, but generally speaking the current configuration is likely +best for everyone's general purposes, so if changes are made please revert them prior to commit. \ No newline at end of file diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml index 7a657880a88..ba037cfd402 100644 --- a/giraph-gremlin/pom.xml +++ b/giraph-gremlin/pom.xml @@ -201,6 +201,10 @@ limitations under the License. + + org.apache.maven.plugins + maven-surefire-plugin + org.apache.maven.plugins maven-failsafe-plugin @@ -240,20 +244,6 @@ limitations under the License. - - org.apache.maven.plugins - maven-surefire-plugin - - - - - **/*IntegrateTest.java - **/*PerformanceTest.java - - **/HadoopGremlinPluginTest.java - - - \ No newline at end of file diff --git a/giraph-gremlin/src/test/resources/log4j-silent.properties b/giraph-gremlin/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/giraph-gremlin/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/giraph-gremlin/src/test/resources/log4j-test.properties b/giraph-gremlin/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..a3e679c0eb3 --- /dev/null +++ b/giraph-gremlin/src/test/resources/log4j-test.properties @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n + +log4j.logger.org.apache.hadoop=ERROR \ No newline at end of file diff --git a/gremlin-console/pom.xml b/gremlin-console/pom.xml index dfa94153e67..419fd7ee625 100644 --- a/gremlin-console/pom.xml +++ b/gremlin-console/pom.xml @@ -144,6 +144,12 @@ limitations under the License. + + + ${basedir}/src/test/resources + + + org.apache.maven.plugins diff --git a/gremlin-console/src/test/resources/log4j-silent.properties b/gremlin-console/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/gremlin-console/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/gremlin-console/src/test/resources/log4j-test.properties b/gremlin-console/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/gremlin-console/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-core/pom.xml b/gremlin-core/pom.xml index 77d826e1ff4..668830a3c46 100644 --- a/gremlin-core/pom.xml +++ b/gremlin-core/pom.xml @@ -102,6 +102,12 @@ limitations under the License. ${basedir}/target ${project.artifactId}-${project.version} + + + ${basedir}/src/test/resources + + + org.apache.maven.plugins diff --git a/gremlin-core/src/test/resources/log4j-silent.properties b/gremlin-core/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/gremlin-core/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/gremlin-core/src/test/resources/log4j-test.properties b/gremlin-core/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/gremlin-core/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-driver/src/test/resources/log4j-silent.properties b/gremlin-driver/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/gremlin-driver/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/gremlin-driver/src/test/resources/log4j-test.properties b/gremlin-driver/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/gremlin-driver/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-groovy/src/test/resources/log4j-silent.properties b/gremlin-groovy/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/gremlin-groovy/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/gremlin-groovy/src/test/resources/log4j-test.properties b/gremlin-groovy/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/gremlin-groovy/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java similarity index 99% rename from gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java rename to gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java index 5b09425e327..cd3a431ffec 100644 --- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthIntegrateOldTest.java +++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/GremlinServerAuthOldIntegrateTest.java @@ -43,7 +43,7 @@ * @see TINKERPOP3-981 */ @Deprecated -public class GremlinServerAuthIntegrateOldTest extends AbstractGremlinServerIntegrationTest { +public class GremlinServerAuthOldIntegrateTest extends AbstractGremlinServerIntegrationTest { /** * Configure specific Gremlin Server settings for specific tests. diff --git a/gremlin-server/src/test/resources/log4j-silent.properties b/gremlin-server/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/gremlin-server/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/gremlin-server/src/test/resources/log4j-test.properties b/gremlin-server/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/gremlin-server/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-test/pom.xml b/gremlin-test/pom.xml index 8ae4e40036f..e9aacadc8f5 100644 --- a/gremlin-test/pom.xml +++ b/gremlin-test/pom.xml @@ -72,6 +72,12 @@ limitations under the License. + + + ${basedir}/src/test/resources + + + org.apache.maven.plugins diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java index 97dfea29238..ab39572aa77 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGremlinTest.java @@ -191,6 +191,12 @@ public void tryCommit(final Graph graph) { graph.tx().commit(); } + /** + * This method does not appear to be used in TinkerPop core. It was used at one time by Neo4j tests, but wasn't + * really a great pattern and was eventually removed. + * + * @deprecated as of 3.1.1-incubating, and is not replaced + */ public void tryRandomCommit(final Graph graph) { if (graph.features().graph().supportsTransactions() && new Random().nextBoolean()) graph.tx().commit(); diff --git a/gremlin-test/src/test/resources/log4j-silent.properties b/gremlin-test/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9d4f1d40836 --- /dev/null +++ b/gremlin-test/src/test/resources/log4j-silent.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF + +# need to turn this on as this test actually tests "logging" - needs INFO at minimum for tests to pass +log4j.logger.org.apache.tinkerpop.gremlin.util.Log4jRecordingAppenderTest=INFO \ No newline at end of file diff --git a/gremlin-test/src/test/resources/log4j-test.properties b/gremlin-test/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..3d02711c39b --- /dev/null +++ b/gremlin-test/src/test/resources/log4j-test.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n + +# need to turn this on as this test actually tests "logging" - needs INFO at minimum for tests to pass +log4j.logger.org.apache.tinkerpop.gremlin.util.Log4jRecordingAppenderTest=INFO \ No newline at end of file diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml index b2f7d699d29..3542275baae 100644 --- a/hadoop-gremlin/pom.xml +++ b/hadoop-gremlin/pom.xml @@ -196,6 +196,10 @@ limitations under the License. + + org.apache.maven.plugins + maven-surefire-plugin + org.apache.maven.plugins maven-failsafe-plugin @@ -222,20 +226,6 @@ limitations under the License. true - - org.apache.maven.plugins - maven-surefire-plugin - - - - - **/*IntegrateTest.java - **/*PerformanceTest.java - - **/HadoopGremlinPluginTest.java - - - \ No newline at end of file diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java similarity index 97% rename from hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java rename to hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java index 0d7da18b593..4108e35c912 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java @@ -44,9 +44,12 @@ import static org.junit.Assert.*; /** + * This is an test that is mean to be used in the context of the {@link HadoopPluginSuite} and shouldn't be + * executed on its own. + * * @author Marko A. Rodriguez (http://markorodriguez.com) */ -public class HadoopGremlinPluginTest extends AbstractGremlinTest { +public class HadoopGremlinPluginCheck extends AbstractGremlinTest { @Before public void setupTest() { diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java index 59ba23cbf2d..6c4cc201242 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopPluginSuite.java @@ -29,6 +29,6 @@ */ public class HadoopPluginSuite extends AbstractGremlinSuite { public HadoopPluginSuite(final Class klass, final RunnerBuilder builder) throws InitializationError { - super(klass, builder, new Class[]{HadoopGremlinPluginTest.class}, new Class[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER); + super(klass, builder, new Class[]{HadoopGremlinPluginCheck.class}, new Class[]{HadoopGremlinPluginCheck.class}, true, TraversalEngine.Type.COMPUTER); } } diff --git a/hadoop-gremlin/src/test/resources/log4j-silent.properties b/hadoop-gremlin/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/hadoop-gremlin/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/hadoop-gremlin/src/test/resources/log4j-test.properties b/hadoop-gremlin/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..4b5af3100dc --- /dev/null +++ b/hadoop-gremlin/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n diff --git a/neo4j-gremlin/pom.xml b/neo4j-gremlin/pom.xml index b8d628946f3..b13c5566208 100644 --- a/neo4j-gremlin/pom.xml +++ b/neo4j-gremlin/pom.xml @@ -64,6 +64,12 @@ limitations under the License. + + + ${basedir}/src/test/resources + + + org.apache.maven.plugins diff --git a/neo4j-gremlin/src/test/resources/log4j-silent.properties b/neo4j-gremlin/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/neo4j-gremlin/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/neo4j-gremlin/src/test/resources/log4j-test.properties b/neo4j-gremlin/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/neo4j-gremlin/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0332cdcdc33..371a1082530 100644 --- a/pom.xml +++ b/pom.xml @@ -127,13 +127,6 @@ limitations under the License. 2.4.1 4.12 - - ${project.basedir}/../gremlin-server/conf/ - log4j.properties 3.0.2 UTF-8 UTF-8 @@ -144,6 +137,15 @@ limitations under the License. 1.10 1.2 2.10.1 + + + file:target/test-classes/log4j-test.properties + file:target/test-classes/log4j-silent.properties + false @@ -316,12 +318,8 @@ limitations under the License. maven-surefire-plugin 2.17 - -Dlog4j.configuration=${log4j.properties} -Dbuild.dir=${project.build.directory} + -Dlog4j.configuration=${log4j-test.properties} -Dbuild.dir=${project.build.directory} - - - ${log4j.properties.dir} - **/*IntegrateTest.java **/*PerformanceTest.java @@ -343,15 +341,11 @@ limitations under the License. **/*IntegrateTest.java ${skipIntegrationTests} - -Dlog4j.configuration=${log4j.properties} -Dhost=localhost -Dport=8182 + -Dlog4j.configuration=${log4j-test.properties} -Dhost=localhost -Dport=8182 -Djub.consumers=CONSOLE,H2 -Djub.db.file=target/performance/h2/benchmarks -Djub.charts.dir=target/performance/charts -Djub.customkey=${project.parent.version} -Dbuild.dir=${project.build.directory} - - - ${log4j.properties.dir} - target/failsafe-reports/failsafe-integration.xml @@ -365,15 +359,11 @@ limitations under the License. **/*PerformanceTest.java ${skipPerformanceTests} - -Dlog4j.configuration=${log4j.properties} -Dhost=localhost -Dport=8182 + -Dlog4j.configuration=${log4j-test.properties} -Dhost=localhost -Dport=8182 -Djub.consumers=CONSOLE,H2 -Djub.db.file=target/performance/h2/benchmarks -Djub.charts.dir=target/performance/charts -Djub.customkey=${project.parent.version} -Dbuild.dir=${project.build.directory} - - - ${log4j.properties.dir} - target/failsafe-reports/failsafe-performance.xml @@ -418,6 +408,7 @@ limitations under the License. 1.0 + org.apache.maven.plugins maven-javadoc-plugin ${javadoc-plugin.version} @@ -914,7 +905,7 @@ limitations under the License. - + ci false @@ -925,6 +916,14 @@ limitations under the License. + + org.apache.maven.plugins + maven-javadoc-plugin + ${javadoc-plugin.version} + + true + + org.apache.maven.plugins maven-surefire-plugin @@ -933,6 +932,8 @@ limitations under the License. true + -Dlog4j.configuration=${log4j-silent.properties} -Dbuild.dir=${project.build.directory} + **/*IntegrateTest.java **/*PerformanceTest.java diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml index 8721c5f152d..63ac42fff84 100644 --- a/spark-gremlin/pom.xml +++ b/spark-gremlin/pom.xml @@ -345,6 +345,10 @@ + + org.apache.maven.plugins + maven-surefire-plugin + org.apache.maven.plugins maven-failsafe-plugin @@ -387,20 +391,6 @@ - - org.apache.maven.plugins - maven-surefire-plugin - - - - - **/*IntegrateTest.java - **/*PerformanceTest.java - - **/HadoopGremlinPluginTest.java - - - \ No newline at end of file diff --git a/spark-gremlin/src/test/resources/log4j-silent.properties b/spark-gremlin/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/spark-gremlin/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/spark-gremlin/src/test/resources/log4j-test.properties b/spark-gremlin/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..fe1cb1edecd --- /dev/null +++ b/spark-gremlin/src/test/resources/log4j-test.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n + +log4j.logger.org.apache.spark=ERROR +log4j.logger.org.spark-project=ERROR +log4j.logger.org.apache.hadoop=ERROR \ No newline at end of file diff --git a/tinkergraph-gremlin/src/test/resources/log4j-silent.properties b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties new file mode 100644 index 00000000000..9c6148d414c --- /dev/null +++ b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=OFF \ No newline at end of file diff --git a/tinkergraph-gremlin/src/test/resources/log4j-test.properties b/tinkergraph-gremlin/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..ef436fe4799 --- /dev/null +++ b/tinkergraph-gremlin/src/test/resources/log4j-test.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=WARN, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file From bdb8015f550e12cdba3b8b4996099b28a96b808c Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Fri, 4 Dec 2015 10:28:42 -0500 Subject: [PATCH 06/17] Added more to the silent logging configs. Seems like not providing an appender was causing errors in the hadoop tests thus forcing maven to choose a different log4j.properties on the path - weird. Adding a default appender (even though logging is OFF) seemed to make the difference and get "silent" working right. --- giraph-gremlin/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-console/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-core/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-driver/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-groovy/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-server/src/test/resources/log4j-silent.properties | 7 ++++++- gremlin-test/src/test/resources/log4j-silent.properties | 7 ++++++- hadoop-gremlin/src/test/resources/log4j-silent.properties | 7 ++++++- neo4j-gremlin/src/test/resources/log4j-silent.properties | 7 ++++++- spark-gremlin/src/test/resources/log4j-silent.properties | 7 ++++++- .../src/test/resources/log4j-silent.properties | 7 ++++++- 11 files changed, 66 insertions(+), 11 deletions(-) diff --git a/giraph-gremlin/src/test/resources/log4j-silent.properties b/giraph-gremlin/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/giraph-gremlin/src/test/resources/log4j-silent.properties +++ b/giraph-gremlin/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-console/src/test/resources/log4j-silent.properties b/gremlin-console/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/gremlin-console/src/test/resources/log4j-silent.properties +++ b/gremlin-console/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-core/src/test/resources/log4j-silent.properties b/gremlin-core/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/gremlin-core/src/test/resources/log4j-silent.properties +++ b/gremlin-core/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-driver/src/test/resources/log4j-silent.properties b/gremlin-driver/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/gremlin-driver/src/test/resources/log4j-silent.properties +++ b/gremlin-driver/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-groovy/src/test/resources/log4j-silent.properties b/gremlin-groovy/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/gremlin-groovy/src/test/resources/log4j-silent.properties +++ b/gremlin-groovy/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-server/src/test/resources/log4j-silent.properties b/gremlin-server/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/gremlin-server/src/test/resources/log4j-silent.properties +++ b/gremlin-server/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/gremlin-test/src/test/resources/log4j-silent.properties b/gremlin-test/src/test/resources/log4j-silent.properties index 9d4f1d40836..ce0647ef22b 100644 --- a/gremlin-test/src/test/resources/log4j-silent.properties +++ b/gremlin-test/src/test/resources/log4j-silent.properties @@ -15,7 +15,12 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n # need to turn this on as this test actually tests "logging" - needs INFO at minimum for tests to pass log4j.logger.org.apache.tinkerpop.gremlin.util.Log4jRecordingAppenderTest=INFO \ No newline at end of file diff --git a/hadoop-gremlin/src/test/resources/log4j-silent.properties b/hadoop-gremlin/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/hadoop-gremlin/src/test/resources/log4j-silent.properties +++ b/hadoop-gremlin/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/neo4j-gremlin/src/test/resources/log4j-silent.properties b/neo4j-gremlin/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/neo4j-gremlin/src/test/resources/log4j-silent.properties +++ b/neo4j-gremlin/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/spark-gremlin/src/test/resources/log4j-silent.properties b/spark-gremlin/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/spark-gremlin/src/test/resources/log4j-silent.properties +++ b/spark-gremlin/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file diff --git a/tinkergraph-gremlin/src/test/resources/log4j-silent.properties b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties index 9c6148d414c..1825bb07a41 100644 --- a/tinkergraph-gremlin/src/test/resources/log4j-silent.properties +++ b/tinkergraph-gremlin/src/test/resources/log4j-silent.properties @@ -15,4 +15,9 @@ # specific language governing permissions and limitations # under the License. -log4j.rootLogger=OFF \ No newline at end of file +# this file should always have logging set to OFF. it seems, however, that an appender of some sort is +# required or else some logs throw error and use other log4j.properties files on the path. +log4j.rootLogger=OFF, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n \ No newline at end of file From 63a189c7b5b4f5df876165f29e848e742fdcf81c Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Fri, 4 Dec 2015 12:53:35 -0500 Subject: [PATCH 07/17] TINKERPOP-1002 Rolled back transaction after asserted exception. This was a simple one-line fix - CTR. --- .../apache/tinkerpop/gremlin/structure/TransactionTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/TransactionTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/TransactionTest.java index 8d23fceb69c..11beb8bb4ee 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/TransactionTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/TransactionTest.java @@ -83,6 +83,9 @@ public void shouldHaveExceptionConsistencyWhenTransactionOpenOnClose() { fail("An exception should be thrown when close behavior is manual and the graph is close with an open transaction"); } catch (Exception ex) { validateException(Transaction.Exceptions.openTransactionsOnClose(), ex); + } finally { + // rollback manually to keep the test clean + g.tx().rollback(); } } From bd0193ddaa3a68607a26d34998766bf439fac392 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Fri, 4 Dec 2015 15:03:48 -0500 Subject: [PATCH 08/17] TINKERPOP-1022 Preload OpProcessors in Gremlin Server. This gets them warm for the initial request that hits them. CTR --- docs/src/reference/gremlin-applications.asciidoc | 3 +++ .../org/apache/tinkerpop/gremlin/server/GremlinServer.java | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/docs/src/reference/gremlin-applications.asciidoc b/docs/src/reference/gremlin-applications.asciidoc index 643d666ed26..ea47342db08 100644 --- a/docs/src/reference/gremlin-applications.asciidoc +++ b/docs/src/reference/gremlin-applications.asciidoc @@ -320,6 +320,9 @@ $ bin/gremlin-server.sh conf/gremlin-server-modern.yaml [INFO] GremlinExecutor - Initialized gremlin-groovy ScriptEngine with scripts/generate-modern.groovy [INFO] ServerGremlinExecutor - Initialized GremlinExecutor and configured ScriptEngines. [INFO] ServerGremlinExecutor - A GraphTraversalSource is now bound to [g] with graphtraversalsource[tinkergraph[vertices:0 edges:0], standard] +[INFO] OpLoader - Adding the standard OpProcessor. +[INFO] OpLoader - Adding the control OpProcessor. +[INFO] OpLoader - Adding the session OpProcessor. [INFO] GremlinServer - Executing start up LifeCycleHook [INFO] Logger$info - Loading 'modern' graph data. [INFO] AbstractChannelizer - Configured application/vnd.gremlin-v1.0+gryo with org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0 diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/GremlinServer.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/GremlinServer.java index c2c18b702d0..97b8e9b1851 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/GremlinServer.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/GremlinServer.java @@ -104,8 +104,14 @@ public GremlinServer(final Settings settings) { }else { workerGroup = new NioEventLoopGroup(settings.threadPoolWorker, threadFactoryWorker); } + serverGremlinExecutor = new ServerGremlinExecutor<>(settings, null, workerGroup, EventLoopGroup.class); gremlinExecutorService = serverGremlinExecutor.getGremlinExecutorService(); + + // force an early load of the OpLoader (even if not used by a Channelizer). this is an expensive operation + // as it uses ServiceLoader and if left uninitialized puts a burden on the first client request that passes + // through an OpProcessor + OpLoader.getProcessors(); } /** From 8ab1bd38f2a7473bf6774ed8ae6fe8591a33fc6e Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Sun, 6 Dec 2015 15:55:54 -0500 Subject: [PATCH 09/17] Moved transaction commit outside of iteration loop. This change ensures that the iteration completes in full with serialization prior to close of transaction. This can cause problems for graphs with strong transaction semantics. CTR. --- .../server/op/AbstractEvalOpProcessor.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java index 50714295ad5..eaa7d2b7d6f 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java @@ -232,16 +232,6 @@ protected void handleIterator(final Context context, final Iterator itty) throws // effect so if the client is "slow" it may simply timeout. if (aggregate.size() < resultIterationBatchSize) aggregate.add(itty.next()); - // if there's no more items in the iterator then we've aggregated everything and are thus ready to - // commit stuff if transaction management is on. exceptions should bubble up and be handle in the normal - // manner of things. a final SUCCESS message will not have been sent (below) and we ship back an error. - // if transaction management is not enabled, then returning SUCCESS below is OK as this is a different - // usage context. without transaction management enabled, the user is responsible for maintaining - // the transaction and will want a SUCCESS to know their eval and iteration was ok. they would then - // potentially have a failure on commit on the next request. - if (!itty.hasNext() && manageTransactions) - context.getGraphManager().commitAll(); - // send back a page of results if batch size is met or if it's the end of the results being iterated. // also check writeability of the channel to prevent OOME for slow clients. if (ctx.channel().isWritable()) { @@ -275,6 +265,15 @@ protected void handleIterator(final Context context, final Iterator itty) throws stopWatch.unsplit(); } + // if there's no more items in the iterator then we've aggregated everything and are thus ready to + // commit stuff if transaction management is on. exceptions should bubble up and be handle in the normal + // manner of things. a final SUCCESS message will not have been sent (below) and we ship back an error. + // if transaction management is not enabled, then returning SUCCESS below is OK as this is a different + // usage context. without transaction management enabled, the user is responsible for maintaining + // the transaction and will want a SUCCESS to know their eval and iteration was ok. they would then + // potentially have a failure on commit on the next request. + if (manageTransactions) context.getGraphManager().commitAll(); + stopWatch.stop(); } From 43b790dd1c8cdd346dded070d3727faf35945afc Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Mon, 7 Dec 2015 13:29:01 -0500 Subject: [PATCH 10/17] TINKERPOP-1018 Fixed some settings that weren't being passed to Gremlin Driver Cluster creation through config file. There were more than just maxContentLength as defined in the issue. --- CHANGELOG.asciidoc | 1 + .../tinkerpop/gremlin/driver/Cluster.java | 24 +++++++++++++++++++ .../tinkerpop/gremlin/driver/Settings.java | 17 +++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9b3fa7174d0..0ebff5e9680 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -27,6 +27,7 @@ TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * Improved logging control during builds with Maven. +* Fixed settings that weren't being passed to the the Gremlin Driver `Cluster` through configuration file. * `Column` now implements `Function` and now `by(valueDecr)` can be replaced with `by(values,decr)` and thus, projection and order are now separated. * `Column` now implements `Function`. The modulator `by(valueDecr)` can be replaced by `by(values,decr)` and thus, projection and order are separated. * Added `InputRDDFormat` which wraps an `InputRDD` to make it accessible to Hadoop and not just Spark. diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java index 7d83c58f0d9..ddc6081da8f 100644 --- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java +++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java @@ -26,6 +26,7 @@ import io.netty.channel.nio.NioEventLoopGroup; import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import javax.net.ssl.TrustManager; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -122,9 +123,17 @@ public static Builder build(final File configurationFile) throws FileNotFoundExc final Builder builder = new Builder(settings.hosts.get(0)) .port(settings.port) .enableSsl(settings.connectionPool.enableSsl) + .trustCertificateChainFile(settings.connectionPool.trustCertChainFile) .nioPoolSize(settings.nioPoolSize) .workerPoolSize(settings.workerPoolSize) + .reconnectInterval(settings.connectionPool.reconnectInterval) + .reconnectIntialDelay(settings.connectionPool.reconnectInitialDelay) + .resultIterationBatchSize(settings.connectionPool.resultIterationBatchSize) + .channelizer(settings.connectionPool.channelizer) + .maxContentLength(settings.connectionPool.maxContentLength) + .maxWaitForConnection(settings.connectionPool.maxWaitForConnection) .maxInProcessPerConnection(settings.connectionPool.maxInProcessPerConnection) + .minInProcessPerConnection(settings.connectionPool.minInProcessPerConnection) .maxSimultaneousUsagePerConnection(settings.connectionPool.maxSimultaneousUsagePerConnection) .minSimultaneousUsagePerConnection(settings.connectionPool.minSimultaneousUsagePerConnection) .maxConnectionPoolSize(settings.connectionPool.maxSize) @@ -133,6 +142,9 @@ public static Builder build(final File configurationFile) throws FileNotFoundExc if (settings.username != null && settings.password != null) builder.credentials(settings.username, settings.password); + if (settings.jaasEntry != null) + builder.jaasEntry(settings.jaasEntry); + // the first address was added above in the constructor, so skip it if there are more if (addresses.size() > 1) addresses.stream().skip(1).forEach(builder::addContactPoint); @@ -249,6 +261,7 @@ public final static class Builder { private int resultIterationBatchSize = Connection.RESULT_ITERATION_BATCH_SIZE; private String channelizer = Channelizer.WebSocketChannelizer.class.getName(); private boolean enableSsl = false; + private String trustCertChainFile = null; private LoadBalancingStrategy loadBalancingStrategy = new LoadBalancingStrategy.RoundRobin(); private AuthProperties authProps = new AuthProperties(); @@ -315,6 +328,16 @@ public Builder enableSsl(final boolean enable) { return this; } + /** + * File location for a SSL Certificate Chain to use when SSL is enabled. If this value is not provided and + * SSL is enabled, the {@link TrustManager} will be established with a self-signed certificate which is NOT + * suitable for production purposes. + */ + public Builder trustCertificateChainFile(final String certificateChainFile) { + this.trustCertChainFile = certificateChainFile; + return this; + } + /** * The minimum number of in-flight requests that can occur on a {@link Connection} before it is considered * for closing on return to the {@link ConnectionPool}. @@ -528,6 +551,7 @@ public Cluster create() { connectionPoolSettings.reconnectInterval = this.reconnectInterval; connectionPoolSettings.resultIterationBatchSize = this.resultIterationBatchSize; connectionPoolSettings.enableSsl = this.enableSsl; + connectionPoolSettings.trustCertChainFile = this.trustCertChainFile; connectionPoolSettings.channelizer = this.channelizer; return new Cluster(getContactPoints(), serializer, this.nioPoolSize, this.workerPoolSize, connectionPoolSettings, loadBalancingStrategy, authProps); diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Settings.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Settings.java index 07f329402c6..7faa377d63b 100644 --- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Settings.java +++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Settings.java @@ -47,9 +47,11 @@ final class Settings { public int workerPoolSize = Runtime.getRuntime().availableProcessors() * 2; - public String username; + public String username = null; - public String password; + public String password = null; + + public String jaasEntry = null; /** * Read configuration from a file into a new {@link Settings} object. @@ -84,8 +86,19 @@ static class ConnectionPoolSettings { public int reconnectInitialDelay = Connection.RECONNECT_INITIAL_DELAY; public int resultIterationBatchSize = Connection.RESULT_ITERATION_BATCH_SIZE; public String channelizer = Channelizer.WebSocketChannelizer.class.getName(); + + /** + * @deprecated as of 3.1.1-incubating, and not replaced as this property was never implemented internally + * as the way to establish sessions + */ + @Deprecated public String sessionId = null; + /** + * @deprecated as of 3.1.1-incubating, and not replaced as this property was never implemented internally + * as the way to establish sessions + */ + @Deprecated public Optional optionalSessionId() { return Optional.ofNullable(sessionId); } From e1a8d7949678858e2fcd061942fe9ab4c7c42bf5 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 13:20:32 -0700 Subject: [PATCH 11/17] hardcoded to not persisted contexts in integration testing to ensure that RDDs are not GC'd. CTR. --- .../spark/process/computer/SparkHadoopGraphProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java index 618904abaa9..b40026e805c 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java @@ -45,7 +45,7 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider { @Override public Map getBaseConfiguration(final String graphName, final Class test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { final Map config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); - config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast + // config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast if (null != loadGraphWith && !test.equals(BulkLoaderVertexProgramTest.class) && RANDOM.nextBoolean()) { From d17224c4a310ed89d15ff04142d36b078d00e677 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 15:48:00 -0700 Subject: [PATCH 12/17] better control over RDD persistence. --- .../process/computer/SparkContextHelper.java | 5 ++- .../process/computer/SparkGraphComputer.java | 3 +- .../spark/structure/io/PersistedInputRDD.java | 31 +------------------ .../structure/io/PersistedOutputRDD.java | 1 + .../gremlin/spark/AbstractSparkTest.java | 3 ++ 5 files changed, 8 insertions(+), 35 deletions(-) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java index 6ceaf9f481a..488186a2965 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java @@ -35,9 +35,8 @@ private SparkContextHelper() { public static void tryToCloseContext(final JavaSparkContext context, final Configuration configuration) { if (context != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false)) { - context.close(); - if (null != Spark.getContext()) - Spark.close(); + Spark.create(context.sc()); + Spark.close(); } } } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java index 21a379a31c9..0c8ac74c348 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java @@ -192,7 +192,6 @@ private Future submitWithExecutor(Executor exec) { hadoopConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, null) != null) && !this.persist.equals(GraphComputer.Persist.NOTHING)) { try { - Spark.removeRDD(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)); // delete existing persisted output rdd hadoopConfiguration.getClass(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD, OutputFormatRDD.class, OutputRDD.class) .newInstance() .writeGraphRDD(apacheConfiguration, graphRDD); @@ -211,7 +210,7 @@ private Future submitWithExecutor(Executor exec) { final JavaPairRDD mapReduceGraphRDD = graphRDD.mapValues(vertexWritable -> { vertexWritable.get().dropEdges(); return vertexWritable; - }).setName("mapReduceGraphRDD").cache(); // drop all the edges of the graph as they are not used in mapReduce processing + }).cache(); // drop all the edges of the graph as they are not used in mapReduce processing for (final MapReduce mapReduce : this.mapReducers) { // execute the map reduce job final HadoopConfiguration newApacheConfiguration = new HadoopConfiguration(apacheConfiguration); diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputRDD.java index a59d556b901..88e1efa8cd1 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputRDD.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputRDD.java @@ -34,39 +34,10 @@ public final class PersistedInputRDD implements InputRDD { @Override public JavaPairRDD readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) { - Spark.create(sparkContext.sc()); final String inputRDDName = configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION, null); if (null == inputRDDName) throw new IllegalArgumentException(PersistedInputRDD.class.getSimpleName() + " requires " + Constants.GREMLIN_HADOOP_INPUT_LOCATION + " in order to retrieve the named graphRDD from the SparkContext"); + Spark.create(sparkContext.sc()); return JavaPairRDD.fromJavaRDD((JavaRDD) Spark.getRDD(inputRDDName).toJavaRDD()); } - - /*public static Optional> getPersistedRDD(final JavaSparkContext sparkContext, final String rddName) { - final Iterator>> iterator = JavaSparkContext.toSparkContext(sparkContext). - getPersistentRDDs(). - toList().iterator(); - while (iterator.hasNext()) { - final Tuple2> tuple2 = iterator.next(); - if (tuple2._2().toString().contains(rddName)) - return Optional.of(tuple2._2()); - } - return Optional.empty(); - } - - public static void removePersistedRDD(final JavaSparkContext sparkContext, final String rddName) { - if (null == rddName) - return; - final List matchingRDDs = new ArrayList<>(); - final Iterator>> iterator = JavaSparkContext.toSparkContext(sparkContext). - getPersistentRDDs(). - toList().iterator(); - while (iterator.hasNext()) { - final Tuple2> tuple2 = iterator.next(); - if (tuple2._2().toString().contains(rddName)) - matchingRDDs.add(tuple2._1()); - } - for (final Object rddId : matchingRDDs) { - JavaSparkContext.toSparkContext(sparkContext).persistentRdds().remove(rddId).get().unpersist(false); - } - }*/ } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedOutputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedOutputRDD.java index 93451b82628..5188f9a9e2d 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedOutputRDD.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedOutputRDD.java @@ -38,6 +38,7 @@ public final class PersistedOutputRDD implements OutputRDD { public void writeGraphRDD(final Configuration configuration, final JavaPairRDD graphRDD) { if (!configuration.containsKey(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)) throw new IllegalArgumentException("There is no provided " + Constants.GREMLIN_HADOOP_OUTPUT_LOCATION + " to write the persisted RDD to"); + Spark.removeRDD(configuration.getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)); // this might be bad cause it unpersists the job RDD if (!configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true)) graphRDD.mapValues(vertex -> { vertex.get().dropEdges(); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java index 02b0e182781..ccff1ab815e 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java @@ -22,6 +22,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.tinkerpop.gremlin.spark.structure.Spark; import org.junit.After; import org.junit.Before; @@ -38,6 +39,8 @@ public void setupTest() { sparkConfiguration.set("spark.master", "local[4]"); JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); sparkContext.close(); + Spark.create(sparkContext.sc()); + Spark.close(); System.out.println("SparkContext has been closed for " + this.getClass().getCanonicalName() + "-setupTest"); } } From 74d1fde2999baea67f91aaa1e2b3e3482db61896 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 16:15:15 -0700 Subject: [PATCH 13/17] no need for SparkContextHelper anymore. Moved everything regarding persitent context management to the static Spark object. --- .../process/computer/SparkContextHelper.java | 42 ------------------- .../process/computer/SparkGraphComputer.java | 2 +- .../gremlin/spark/structure/Spark.java | 22 ++++++---- .../spark/structure/io/InputRDDFormat.java | 5 ++- .../io/PersistedInputOutputRDDTest.java | 34 ++++++--------- 5 files changed, 29 insertions(+), 76 deletions(-) delete mode 100644 spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java deleted file mode 100644 index 488186a2965..00000000000 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkContextHelper.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.spark.process.computer; - -import org.apache.commons.configuration.Configuration; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.tinkerpop.gremlin.hadoop.Constants; -import org.apache.tinkerpop.gremlin.spark.structure.Spark; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -public final class SparkContextHelper { - - private SparkContextHelper() { - - } - - public static void tryToCloseContext(final JavaSparkContext context, final Configuration configuration) { - if (context != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false)) { - Spark.create(context.sc()); - Spark.close(); - } - } -} diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java index 0c8ac74c348..079b0d6f7c3 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java @@ -235,7 +235,7 @@ private Future submitWithExecutor(Executor exec) { finalMemory.setRuntime(System.currentTimeMillis() - startTime); return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), finalMemory.asImmutable()); } finally { - SparkContextHelper.tryToCloseContext(sparkContext, apacheConfiguration); + Spark.tryAndClose(apacheConfiguration); } }, exec); } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java index 5a13593c647..a889c6de430 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java @@ -23,6 +23,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.rdd.RDD; +import org.apache.tinkerpop.gremlin.hadoop.Constants; import scala.collection.JavaConversions; import java.util.ArrayList; @@ -41,14 +42,6 @@ public class Spark { private Spark() { } - private Spark(final SparkContext sparkContext) { - CONTEXT = sparkContext; - } - - public static SparkContext getContext() { - return CONTEXT; - } - public static void create(final Configuration configuration) { final SparkConf sparkConf = new SparkConf(); configuration.getKeys().forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); @@ -60,6 +53,10 @@ public static void create(final SparkContext sparkContext) { CONTEXT = sparkContext; } + public static SparkContext getContext() { + return CONTEXT; + } + public static void refresh() { for (final RDD rdd : JavaConversions.asJavaIterable(CONTEXT.persistentRdds().values())) { if (null != rdd.name()) { @@ -95,7 +92,14 @@ public static void removeRDD(final String name) { } public static void close() { - CONTEXT.stop(); + if (null != CONTEXT) + CONTEXT.stop(); NAME_TO_RDD.clear(); } + + public static void tryAndClose(final Configuration configuration) { + if (CONTEXT != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false)) { + Spark.close(); + } + } } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java index d1a198e2aed..9182b4ce090 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java @@ -31,7 +31,7 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil; -import org.apache.tinkerpop.gremlin.spark.process.computer.SparkContextHelper; +import org.apache.tinkerpop.gremlin.spark.structure.Spark; import scala.Tuple2; import java.io.IOException; @@ -73,6 +73,7 @@ public RecordReader createRecordReader(final Input hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue())); final InputRDD inputRDD = (InputRDD) Class.forName(sparkConfiguration.get(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD)).newInstance(); final JavaSparkContext javaSparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); + Spark.create(javaSparkContext.sc()); final Iterator> iterator = inputRDD.readGraphRDD(ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration()), javaSparkContext).toLocalIterator(); return new RecordReader() { @Override @@ -102,7 +103,7 @@ public float getProgress() throws IOException, InterruptedException { @Override public void close() throws IOException { - SparkContextHelper.tryToCloseContext(javaSparkContext, ConfUtil.makeApacheConfiguration(hadoopConfiguration)); + Spark.tryAndClose(ConfUtil.makeApacheConfiguration(hadoopConfiguration)); } }; } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) { diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java index 13aa2a92fc5..a85200a64bc 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java @@ -76,12 +76,9 @@ public void shouldNotPersistRDDAcrossJobs() throws Exception { "gremlin-groovy", "g.V()").create(graph)).submit().get(); //////// - SparkConf sparkConfiguration = new SparkConf(); - sparkConfiguration.setAppName("shouldNotPersistRDDAcrossJobs"); - ConfUtil.makeHadoopConfiguration(configuration).forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue())); - JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); - Spark.create(sparkContext.sc()); + Spark.create(configuration); assertFalse(Spark.hasRDD(rddName)); + Spark.close(); } @Test @@ -106,11 +103,7 @@ public void shouldPersistRDDAcrossJobs() throws Exception { "gremlin-groovy", "g.V()").create(graph)).submit().get(); //////// - SparkConf sparkConfiguration = new SparkConf(); - sparkConfiguration.setAppName("shouldPersistRDDAcrossJobs"); - ConfUtil.makeHadoopConfiguration(configuration).forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue())); - JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); - Spark.create(sparkContext.sc()); + Spark.create(configuration); assertTrue(Spark.hasRDD(rddName)); /////// configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, PersistedInputRDD.class.getCanonicalName()); @@ -125,6 +118,7 @@ public void shouldPersistRDDAcrossJobs() throws Exception { .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)), "gremlin-groovy", "g.V()").create(graph)).submit().get(); + Spark.close(); } @Test @@ -157,10 +151,7 @@ public void testBulkLoaderVertexProgramChain() throws Exception { .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration).create(bulkLoaderGraph)) .submit().get(); //// - SparkConf sparkConfiguration = new SparkConf(); - sparkConfiguration.setAppName("testBulkLoaderVertexProgramChain"); - JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); - Spark.create(sparkContext.sc()); + Spark.create(readConfiguration); assertTrue(Spark.hasRDD(rddName)); //// final Graph graph = TinkerGraph.open(); @@ -171,6 +162,8 @@ public void testBulkLoaderVertexProgramChain() throws Exception { assertEquals("marko", g.V().has("name", "marko").values("name").next()); assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue()); assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue()); + //// + Spark.close(); } @Test @@ -199,10 +192,7 @@ public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throw .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration).create(bulkLoaderGraph)) .submit().get(); //// - SparkConf sparkConfiguration = new SparkConf(); - sparkConfiguration.setAppName("testBulkLoaderVertexProgramChain"); - JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); - Spark.create(sparkContext.sc()); + Spark.create(readConfiguration); assertTrue(Spark.hasRDD(rddName)); //// final Graph graph = TinkerGraph.open(); @@ -213,6 +203,8 @@ public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throw assertEquals("marko", g.V().has("name", "marko").values("name").next()); assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue()); assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue()); + //// + Spark.close(); } @Test @@ -228,6 +220,7 @@ public void testComplexChain() throws Exception { configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); + Spark.create(configuration); Graph graph = GraphFactory.open(configuration); graph = graph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.EDGES).program(PageRankVertexProgram.build().iterations(2).create(graph)).submit().get().graph(); GraphTraversalSource g = graph.traversal(); @@ -236,10 +229,6 @@ public void testComplexChain() throws Exception { assertEquals(6l, g.V().values(PageRankVertexProgram.PAGE_RANK).count().next().longValue()); assertEquals(6l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue()); //// - SparkConf sparkConfiguration = new SparkConf(); - sparkConfiguration.setAppName("testComplexChain"); - JavaSparkContext sparkContext = new JavaSparkContext(SparkContext.getOrCreate(sparkConfiguration)); - Spark.create(sparkContext.sc()); assertTrue(Spark.hasRDD(rddName)); //// configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, PersistedInputRDD.class.getCanonicalName()); @@ -277,5 +266,6 @@ public void testComplexChain() throws Exception { assertEquals(0l, g.V().values(PageRankVertexProgram.EDGE_COUNT).count().next().longValue()); //// assertFalse(Spark.hasRDD(rddName)); + Spark.close(); } } From 8d7b8533cc3a236b6423561d52beb3d8cb2f71a8 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 16:50:19 -0700 Subject: [PATCH 14/17] minor tweaks. Payload null check (can happen with persistent RDDs...) and clear out Spark cache for stale unpersited RDDs. --- .../process/computer/payload/ViewIncomingPayload.java | 2 ++ .../apache/tinkerpop/gremlin/spark/structure/Spark.java | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java index a2c92057ea6..2dc9796884e 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java @@ -83,6 +83,8 @@ private void mergeViewIncomingPayload(final ViewIncomingPayload viewIncomingP } public void mergePayload(final Payload payload, final MessageCombiner messageCombiner) { + if (null == payload) + return; if (payload instanceof ViewPayload) this.view = ((ViewPayload) payload).getView(); else if (payload instanceof MessagePayload) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java index a889c6de430..62b0ab319c8 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java @@ -27,9 +27,12 @@ import scala.collection.JavaConversions; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; /** * @author Marko A. Rodriguez (http://markorodriguez.com) @@ -58,11 +61,15 @@ public static SparkContext getContext() { } public static void refresh() { + final Set keepNames = new HashSet<>(); for (final RDD rdd : JavaConversions.asJavaIterable(CONTEXT.persistentRdds().values())) { if (null != rdd.name()) { + keepNames.add(rdd.name()); NAME_TO_RDD.put(rdd.name(), rdd); } } + // remove all stale names in the NAME_TO_RDD map + NAME_TO_RDD.keySet().stream().filter(key -> !keepNames.contains(key)).collect(Collectors.toList()).forEach(NAME_TO_RDD::remove); } public static RDD getRDD(final String name) { From 5a3c5fbc67e0e77d54e95cee456579e9438ef92b Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 17:36:55 -0700 Subject: [PATCH 15/17] various clean ups and organizations. This static Spark class was the way to go. Things now 'just work'. Finally. --- .../process/computer/SparkGraphComputer.java | 3 ++- .../tinkerpop/gremlin/spark/structure/Spark.java | 16 ++++++++-------- .../spark/structure/io/InputRDDFormat.java | 2 +- .../io/PersistedInputOutputRDDTest.java | 12 ++++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java index 079b0d6f7c3..2b85036d4c3 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java @@ -235,7 +235,8 @@ private Future submitWithExecutor(Executor exec) { finalMemory.setRuntime(System.currentTimeMillis() - startTime); return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), finalMemory.asImmutable()); } finally { - Spark.tryAndClose(apacheConfiguration); + if (!apacheConfiguration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false)) + Spark.close(); } }, exec); } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java index 62b0ab319c8..cc1d2bfcda4 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/Spark.java @@ -23,7 +23,6 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.rdd.RDD; -import org.apache.tinkerpop.gremlin.hadoop.Constants; import scala.collection.JavaConversions; import java.util.ArrayList; @@ -48,7 +47,14 @@ private Spark() { public static void create(final Configuration configuration) { final SparkConf sparkConf = new SparkConf(); configuration.getKeys().forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); - sparkConf.setAppName("Spark RDD Utility"); + sparkConf.setAppName("Spark-Gremlin Persisted Context Application"); + CONTEXT = SparkContext.getOrCreate(sparkConf); + } + + public static void create(final String master) { + final SparkConf sparkConf = new SparkConf(); + sparkConf.setAppName("Spark-Gremlin Persisted Context Application"); + sparkConf.setMaster(master); CONTEXT = SparkContext.getOrCreate(sparkConf); } @@ -103,10 +109,4 @@ public static void close() { CONTEXT.stop(); NAME_TO_RDD.clear(); } - - public static void tryAndClose(final Configuration configuration) { - if (CONTEXT != null && !configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false)) { - Spark.close(); - } - } } diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java index 9182b4ce090..d00af5563d3 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDFormat.java @@ -103,7 +103,7 @@ public float getProgress() throws IOException, InterruptedException { @Override public void close() throws IOException { - Spark.tryAndClose(ConfUtil.makeApacheConfiguration(hadoopConfiguration)); + } }; } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) { diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java index a85200a64bc..9ad50a9c201 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDTest.java @@ -56,6 +56,7 @@ public class PersistedInputOutputRDDTest extends AbstractSparkTest { @Test public void shouldNotPersistRDDAcrossJobs() throws Exception { + Spark.create("local[4]"); final String rddName = "target/test-output/" + UUID.randomUUID(); final Configuration configuration = new BaseConfiguration(); configuration.setProperty("spark.master", "local[4]"); @@ -76,7 +77,7 @@ public void shouldNotPersistRDDAcrossJobs() throws Exception { "gremlin-groovy", "g.V()").create(graph)).submit().get(); //////// - Spark.create(configuration); + Spark.create("local[4]"); assertFalse(Spark.hasRDD(rddName)); Spark.close(); } @@ -103,7 +104,6 @@ public void shouldPersistRDDAcrossJobs() throws Exception { "gremlin-groovy", "g.V()").create(graph)).submit().get(); //////// - Spark.create(configuration); assertTrue(Spark.hasRDD(rddName)); /////// configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, PersistedInputRDD.class.getCanonicalName()); @@ -123,6 +123,8 @@ public void shouldPersistRDDAcrossJobs() throws Exception { @Test public void testBulkLoaderVertexProgramChain() throws Exception { + Spark.create("local[4]"); + final String rddName = "target/test-output/" + UUID.randomUUID().toString(); final Configuration readConfiguration = new BaseConfiguration(); readConfiguration.setProperty("spark.master", "local[4]"); @@ -151,7 +153,6 @@ public void testBulkLoaderVertexProgramChain() throws Exception { .program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph(writeConfiguration).create(bulkLoaderGraph)) .submit().get(); //// - Spark.create(readConfiguration); assertTrue(Spark.hasRDD(rddName)); //// final Graph graph = TinkerGraph.open(); @@ -168,6 +169,8 @@ public void testBulkLoaderVertexProgramChain() throws Exception { @Test public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throws Exception { + Spark.create("local[4]"); + final String rddName = "target/test-output/" + UUID.randomUUID().toString(); final Configuration readConfiguration = new BaseConfiguration(); readConfiguration.setProperty("spark.master", "local[4]"); @@ -209,6 +212,8 @@ public void testBulkLoaderVertexProgramChainWithInputOutputHelperMapping() throw @Test public void testComplexChain() throws Exception { + Spark.create("local[4]"); + final String rddName = "target/test-output/graphRDD"; final Configuration configuration = new BaseConfiguration(); configuration.setProperty("spark.master", "local[4]"); @@ -220,7 +225,6 @@ public void testComplexChain() throws Exception { configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); - Spark.create(configuration); Graph graph = GraphFactory.open(configuration); graph = graph.compute(SparkGraphComputer.class).persist(GraphComputer.Persist.EDGES).program(PageRankVertexProgram.build().iterations(2).create(graph)).submit().get().graph(); GraphTraversalSource g = graph.traversal(); From 0d4f01d7b58ea5f20746e6614d25eb54500352d6 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 7 Dec 2015 17:40:54 -0700 Subject: [PATCH 16/17] merged master but then use persisted contexts. --- .../spark/process/computer/SparkHadoopGraphProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java index 60b8fe4b6df..f0ba08f91a2 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java @@ -43,7 +43,7 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider { @Override public Map getBaseConfiguration(final String graphName, final Class test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { final Map config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); - // config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast + config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast if (null != loadGraphWith && !test.equals(BulkLoaderVertexProgramTest.class) && RANDOM.nextBoolean()) { From c94ecd0973a39313c7c3a45a220bc410ea0536b7 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Tue, 8 Dec 2015 06:44:47 -0700 Subject: [PATCH 17/17] removed wildcard import on statics. --- .../apache/tinkerpop/gremlin/spark/structure/SparkTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java index ea1fb2ecefb..9e9882cc7c1 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java @@ -37,7 +37,10 @@ import org.junit.Test; import scala.collection.JavaConversions; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * @author Marko A. Rodriguez (http://markorodriguez.com)