Skip to content
Permalink
Browse files

Merge pull request #67 from mattcasters/master

Update to 3.15.0
  • Loading branch information...
mattcasters committed Nov 17, 2018
2 parents ff2f5c8 + d78d087 commit 44b390159df2e105aab6d9c95adfc320a9b281de
Showing with 547 additions and 70 deletions.
  1. +1 −1 pom.xml
  2. +7 −0 src/main/java/bi/know/kettle/neo4j/core/GraphUsage.java
  3. +6 −2 src/main/java/bi/know/kettle/neo4j/core/MetaStoreUtil.java
  4. +5 −0 src/main/java/bi/know/kettle/neo4j/core/Neo4jDefaults.java
  5. +15 −1 src/main/java/bi/know/kettle/neo4j/entries/check/CheckConnections.java
  6. +16 −7 src/main/java/bi/know/kettle/neo4j/entries/cypherscript/CypherScript.java
  7. +3 −1 src/main/java/bi/know/kettle/neo4j/model/GraphModelDialog.java
  8. +2 −0 src/main/java/bi/know/kettle/neo4j/model/GraphPropertyType.java
  9. +115 −0 src/main/java/bi/know/kettle/neo4j/perspective/Neo4jHelper.java
  10. +1 −0 src/main/java/bi/know/kettle/neo4j/perspective/Neo4jSpoonPlugin.java
  11. +73 −0 src/main/java/bi/know/kettle/neo4j/shared/DriverSingleton.java
  12. +2 −0 src/main/java/bi/know/kettle/neo4j/shared/NeoConnectionDialog.java
  13. +45 −0 src/main/java/bi/know/kettle/neo4j/steps/BaseNeoStep.java
  14. +17 −0 src/main/java/bi/know/kettle/neo4j/steps/BaseNeoStepData.java
  15. +5 −8 src/main/java/bi/know/kettle/neo4j/steps/cypher/Cypher.java
  16. +53 −18 src/main/java/bi/know/kettle/neo4j/steps/graph/GraphOutput.java
  17. +2 −3 src/main/java/bi/know/kettle/neo4j/steps/graph/GraphOutputData.java
  18. +57 −19 src/main/java/bi/know/kettle/neo4j/steps/output/Neo4JOutput.java
  19. +4 −2 src/main/java/bi/know/kettle/neo4j/steps/output/Neo4JOutputData.java
  20. +13 −8 src/main/java/bi/know/kettle/neo4j/steps/output/Neo4JOutputDialog.java
  21. +45 −0 src/main/java/bi/know/kettle/neo4j/xp/CloseDriverSingletonInTopLevelJobExtensionPoint.java
  22. +43 −0 ...main/java/bi/know/kettle/neo4j/xp/CloseDriverSingletonInTopLevelTransformationExtensionPoint.java
  23. +17 −0 src/main/resources/neo4j_spoon_overlays.xul
@@ -4,7 +4,7 @@

<groupId>bi.know.kettle.neo4j.output</groupId>
<artifactId>Neo4JOutput</artifactId>
<version>3.14.3-SNAPSHOT</version>
<version>3.15.0-SNAPSHOT</version>
<packaging>jar</packaging>

<name>Neo4JOutput</name>
@@ -0,0 +1,7 @@
package bi.know.kettle.neo4j.core;

public enum GraphUsage {
NODE_CREATE, NODE_UPDATE, NODE_DELETE, NODE_READ,
RELATIONSHIP_CREATE, RELATIONSHIP_UPDATE, RELATIONSHIP_DELETE, RELATIONSHIP_READ
;
}
@@ -2,13 +2,15 @@

import org.pentaho.di.core.logging.LoggingObjectInterface;
import org.pentaho.di.job.Job;
import org.pentaho.di.metastore.MetaStoreConst;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.metastore.api.IMetaStore;
import org.pentaho.metastore.api.exceptions.MetaStoreException;

public class MetaStoreUtil {

public static final IMetaStore findMetaStore( LoggingObjectInterface executor ) {
public static final IMetaStore findMetaStore( LoggingObjectInterface executor ) throws MetaStoreException {

if ( executor instanceof StepInterface ) {
StepInterface step = (StepInterface) executor;
@@ -53,6 +55,8 @@ public static final IMetaStore findMetaStore( LoggingObjectInterface executor )

// Didn't find it anywhere in the tree above: lazy programmers!
//
return null;
System.err.println("METASTORE PROBLEM: Local couldn't be found, force load local");

return MetaStoreConst.openLocalPentahoMetaStore();
}
}
@@ -2,4 +2,9 @@

public class Neo4jDefaults {
public static String NAMESPACE = "Neo4j";

public static String TRANS_NODE_UPDATES_GROUP = "NODE_UPDATES";

public static String VARIABLE_NEO4J_CLEANUP_DRIVERS = "NEO4J_CLEANUP_DRIVERS";

}
@@ -1,13 +1,18 @@
package bi.know.kettle.neo4j.entries.check;

import bi.know.kettle.neo4j.core.MetaStoreUtil;
import bi.know.kettle.neo4j.core.Neo4jDefaults;
import bi.know.kettle.neo4j.shared.DriverSingleton;
import bi.know.kettle.neo4j.shared.NeoConnection;
import org.neo4j.driver.v1.Driver;
import org.neo4j.driver.v1.Session;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.annotations.JobEntry;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.logging.LogChannel;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.job.entry.JobEntryBase;
import org.pentaho.di.job.entry.JobEntryInterface;
@@ -90,6 +95,11 @@ public CheckConnections( String name, String description ) {

@Override public Result execute( Result result, int nr ) throws KettleException {

try {
metaStore = MetaStoreUtil.findMetaStore( this );
} catch(Exception e) {
throw new KettleException( "Error finding metastore", e );
}
MetaStoreFactory<NeoConnection> connectionFactory = new MetaStoreFactory<>( NeoConnection.class, metaStore, Neo4jDefaults.NAMESPACE );

// Replace variables & parameters
@@ -111,7 +121,11 @@ public CheckConnections( String name, String description ) {
throw new KettleException( "Unable to find connection with name '"+connectionName+"'" );
}
connection.initializeVariablesFrom( this );
connection.test();

Driver driver = DriverSingleton.getDriver( log, connection);
Session session = driver.session();
session.close();

} catch(Exception e) {
// Something bad happened, log the error, flag error
//
@@ -1,6 +1,8 @@
package bi.know.kettle.neo4j.entries.cypherscript;

import bi.know.kettle.neo4j.core.MetaStoreUtil;
import bi.know.kettle.neo4j.core.Neo4jDefaults;
import bi.know.kettle.neo4j.shared.DriverSingleton;
import bi.know.kettle.neo4j.shared.NeoConnection;
import org.apache.commons.lang.StringUtils;
import org.neo4j.driver.v1.Driver;
@@ -88,20 +90,30 @@ public CypherScript( String name, String description ) {

@Override public Result execute( Result result, int nr ) throws KettleException {

try {
metaStore = MetaStoreUtil.findMetaStore( this );
} catch(Exception e) {
throw new KettleException( "Error finding metastore", e );
}
MetaStoreFactory<NeoConnection> connectionFactory = new MetaStoreFactory<>( NeoConnection.class, metaStore, Neo4jDefaults.NAMESPACE );

// Replace variables & parameters
//
NeoConnection connection;
String realConnectionName = environmentSubstitute( connectionName );
try {
connection = connectionFactory.loadElement( environmentSubstitute( connectionName ) );
if (StringUtils.isEmpty( realConnectionName )) {
throw new KettleException( "The Neo4j connection name is not set" );
}

connection = connectionFactory.loadElement( realConnectionName );
if (connection==null) {
throw new KettleException( "Unable to find connection with name '"+connectionName+"'" );
throw new KettleException( "Unable to find connection with name '"+realConnectionName+"'" );
}
} catch(Exception e) {
result.setResult( false );
result.increaseErrors( 1L );
throw new KettleException( "Unable to load or find connection with name '"+connectionName+"'", e);
throw new KettleException( "Unable to load or find connection with name '"+realConnectionName+"'", e);
}

String realScript;
@@ -123,7 +135,7 @@ public CypherScript( String name, String description ) {

// Connect to the database
//
driver = connection.getDriver( log );
driver = DriverSingleton.getDriver( log, connection );
session = driver.session();
transaction = session.beginTransaction();

@@ -167,9 +179,6 @@ public CypherScript( String name, String description ) {
if (session!=null) {
session.close();
}
if (driver!=null) {
driver.close();
}
}

if (result.getNrErrors()==0) {
@@ -1,5 +1,6 @@
package bi.know.kettle.neo4j.model;

import bi.know.kettle.neo4j.steps.output.Neo4JOutputDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.custom.CTabFolder;
@@ -751,7 +752,8 @@ private void importNodeProperties() {
break;
}

activeNode.getProperties().add( new GraphProperty( field, "", propertyType, false ) );
String propertyName = Neo4JOutputDialog.standardizePropertyName( valueMeta );
activeNode.getProperties().add( new GraphProperty( propertyName, "", propertyType, false ) );
}
refreshNodeFields();
}
@@ -113,6 +113,8 @@ public static final GraphPropertyType getTypeFromKettle( ValueMetaInterface valu
return GraphPropertyType.ByteArray;
case ValueMetaInterface.TYPE_BIGNUMBER:
return GraphPropertyType.String;
case ValueMetaInterface.TYPE_INTEGER:
return GraphPropertyType.Integer;
default:
return GraphPropertyType.String;
}
@@ -0,0 +1,115 @@
/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package bi.know.kettle.neo4j.perspective;

import bi.know.kettle.neo4j.core.Neo4jDefaults;
import bi.know.kettle.neo4j.shared.NeoConnection;
import bi.know.kettle.neo4j.shared.NeoConnectionUtils;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.ui.core.dialog.EnterSelectionDialog;
import org.pentaho.di.ui.core.dialog.ErrorDialog;
import org.pentaho.di.ui.spoon.ISpoonMenuController;
import org.pentaho.di.ui.spoon.Spoon;
import org.pentaho.di.ui.spoon.dialog.MetaStoreExplorerDialog;
import org.pentaho.metastore.persist.MetaStoreFactory;
import org.pentaho.ui.xul.dom.Document;
import org.pentaho.ui.xul.impl.AbstractXulEventHandler;

import java.util.Collections;
import java.util.List;

public class Neo4jHelper extends AbstractXulEventHandler implements ISpoonMenuController {
protected static Class<?> PKG = Neo4jHelper.class; // for i18n

private static Neo4jHelper instance = null;

private Spoon spoon;
private MetaStoreFactory<NeoConnection> connectionFactory;
private VariableSpace space;

private Neo4jHelper() {
spoon = Spoon.getInstance();
}

public static Neo4jHelper getInstance() {
if ( instance == null ) {
instance = new Neo4jHelper(); ;
instance.spoon.addSpoonMenuController( instance );
instance.space = new Variables();
instance.space.initializeVariablesFrom( null );
instance.connectionFactory = new MetaStoreFactory<NeoConnection>( NeoConnection.class, instance.spoon.getMetaStore(), Neo4jDefaults.NAMESPACE );
}
return instance;
}

public String getName() {
return "neo4jHelper";
}

public void updateMenu( Document doc ) {
// Nothing so far.
}

public void createConnection() {
NeoConnectionUtils.newConnection( spoon.getShell(), space, connectionFactory );
}

public void editConnection() {
try {
List<String> elementNames = connectionFactory.getElementNames();
Collections.sort(elementNames);
String[] names = elementNames.toArray( new String[ 0 ] );

EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Edit Neo4j connection", "Select the connection to edit" );
String choice = dialog.open();
if (choice!=null) {
NeoConnectionUtils.editConnection( spoon.getShell(), space, connectionFactory, choice );
}
} catch(Exception e) {
new ErrorDialog( spoon.getShell(), "Error", "Error editing Neo4j connection", e );
}
}

public void deleteConnection() {
try {
List<String> elementNames = connectionFactory.getElementNames();
Collections.sort( elementNames );
String[] names = elementNames.toArray( new String[ 0 ] );

EnterSelectionDialog dialog = new EnterSelectionDialog( spoon.getShell(), names, "Edit Neo4j connection", "Select the connection to edit" );
String choice = dialog.open();
if ( choice != null ) {
NeoConnectionUtils.deleteConnection( spoon.getShell(), connectionFactory, choice );
}
} catch(Exception e) {
new ErrorDialog( spoon.getShell(), "Error", "Error deleting Neo4j connection", e );
}
}

public void showMetaStoreBrowser() {
Spoon spoon = Spoon.getInstance();
new MetaStoreExplorerDialog( spoon.getShell(), spoon.getMetaStore() ).open();
}

}
@@ -50,6 +50,7 @@ public void applyToContainer( String category, XulDomContainer container ) throw
container.registerClassLoader( getClass().getClassLoader() );
if ( category.equals( "spoon" ) ) {
container.loadOverlay( "neo4j_spoon_overlays.xul", resourceBundle );
container.addEventHandler( Neo4jHelper.getInstance() );
}
}

@@ -0,0 +1,73 @@
package bi.know.kettle.neo4j.shared;

import org.neo4j.driver.v1.Driver;
import org.pentaho.di.core.logging.LogChannelInterface;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DriverSingleton {

private static DriverSingleton singleton;

private Map<String, org.neo4j.driver.v1.Driver> driverMap;

private DriverSingleton() {
driverMap = new HashMap<>( );
}

public static DriverSingleton getInstance() {
if (singleton==null) {
singleton = new DriverSingleton();
}
return singleton;
}

public static Driver getDriver( LogChannelInterface log, NeoConnection connection) {
DriverSingleton ds = getInstance();

String key = getDriverKey( connection );

Driver driver = ds.driverMap.get(key);
if (driver==null) {
driver = connection.getDriver( log );
ds.driverMap.put(key, driver);
}

return driver;
}

public static void closeAll() {
DriverSingleton ds = getInstance();

List<String> keys = new ArrayList<>( ds.getDriverMap().keySet() );
for (String key : keys) {
synchronized ( ds.getDriverMap() ) {
Driver driver = ds.driverMap.get( key );
driver.close();
ds.driverMap.remove( key );
}
}
}

private static String getDriverKey(NeoConnection connection) {
String hostname = connection.environmentSubstitute( connection.getServer() );
String boltPort = connection.environmentSubstitute( connection.getBoltPort() );
String username = connection.environmentSubstitute( connection.getUsername() );

return hostname+":"+boltPort+"@"+username;
}

/**
* Gets driverMap
*
* @return value of driverMap
*/
public Map<String, Driver> getDriverMap() {
return driverMap;
}


}
@@ -324,6 +324,7 @@ public void getData() {
wName.setText( Const.NVL( neoConnection.getName(), "" ) );
wServer.setText( Const.NVL( neoConnection.getServer(), "" ) );
wBoltPort.setText( Const.NVL( neoConnection.getBoltPort(), "" ) );
wBrowserPort.setText( Const.NVL( neoConnection.getBrowserPort(), "" ) );
wRouting.setSelection( neoConnection.isRouting() );
wPolicy.setText( Const.NVL( neoConnection.getRoutingPolicy(), "" ) );
wUsername.setText( Const.NVL( neoConnection.getUsername(), "" ) );
@@ -357,6 +358,7 @@ private void getInfo( NeoConnection neo ) {
neo.setName( wName.getText() );
neo.setServer( wServer.getText() );
neo.setBoltPort( wBoltPort.getText() );
neo.setBrowserPort( wBrowserPort.getText() );
neo.setRouting( wRouting.getSelection() );
neo.setRoutingPolicy( wPolicy.getText() );
neo.setUsername( wUsername.getText() );

0 comments on commit 44b3901

Please sign in to comment.
You can’t perform that action at this time.