Skip to content

Commit

Permalink
[BACKLOG-9928] Improve XMI generation to use DatabaseMetadata (#2819)
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-cuellar @mdamour1976
e-cuellar authored and mdamour1976 committed Aug 17, 2016
1 parent b6c4f1d commit 4ebd088
Show file tree
Hide file tree
Showing 2 changed files with 163 additions and 32 deletions.
99 changes: 74 additions & 25 deletions core/src/org/pentaho/di/core/database/Database.java
View file
Expand Up @@ -2132,31 +2132,15 @@ public RowMetaInterface getQueryFields( String sql, boolean param, RowMetaInterf
// For now, we just try to get the field layout on the re-bound in the
// exception block below.
//
if ( databaseMeta.supportsPreparedStatementMetadataRetrieval() ) {
// On with the regular program.
//

PreparedStatement preparedStatement = null;
try {
preparedStatement =
connection.prepareStatement(
databaseMeta.stripCR( sql ), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY );
preparedStatement.setMaxRows( 1 );
ResultSetMetaData rsmd = preparedStatement.getMetaData();
fields = getRowInfo( rsmd, false, false );
} catch ( Exception e ) {
fields = getQueryFieldsFallback( sql, param, inform, data );
} finally {
if ( preparedStatement != null ) {
try {
preparedStatement.close();
} catch ( SQLException e ) {
throw new KettleDatabaseException(
"Unable to close prepared statement after determining SQL layout", e );
}
}
try {
if ( databaseMeta.supportsPreparedStatementMetadataRetrieval() ) {
// On with the regular program.
//
fields = getQueryFieldsFromPreparedStatement( sql );
} else {
fields = getQueryFieldsFromDatabaseMetaData();
}
} else {
} catch ( Exception e ) {
/*
* databaseMeta.getDatabaseType()==DatabaseMeta.TYPE_DATABASE_SYBASEIQ ) {
*/
Expand All @@ -2173,7 +2157,72 @@ public RowMetaInterface getQueryFields( String sql, boolean param, RowMetaInterf
return fields;
}

private RowMetaInterface getQueryFieldsFallback( String sql, boolean param, RowMetaInterface inform,
public RowMetaInterface getQueryFieldsFromPreparedStatement( String sql ) throws Exception {
PreparedStatement preparedStatement = null;
try {
preparedStatement =
connection.prepareStatement( databaseMeta.stripCR( sql ), ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY );
preparedStatement.setMaxRows( 1 );
ResultSetMetaData rsmd = preparedStatement.getMetaData();
return getRowInfo( rsmd, false, false );
} catch ( Exception e ) {
throw new Exception( e );
} finally {
if ( preparedStatement != null ) {
try {
preparedStatement.close();
} catch ( SQLException e ) {
throw new KettleDatabaseException( "Unable to close prepared statement after determining SQL layout", e );
}
}
}
}

public RowMetaInterface getQueryFieldsFromDatabaseMetaData() throws Exception {

ResultSet columns = connection.getMetaData().getColumns( "", "", databaseMeta.getName(), "" );
RowMetaInterface rowMeta = new RowMeta();
while ( columns.next() ) {
ValueMetaInterface valueMeta = null;
String name = columns.getString( "COLUMN_NAME" );
String type = columns.getString( "SOURCE_DATA_TYPE" );
int size = columns.getInt( "COLUMN_SIZE" );
if ( type.equals( "Integer" ) ) {
valueMeta = new ValueMetaInteger();
} else if ( type.equals( "BigDecimal" ) ) {
valueMeta = new ValueMetaBigNumber();
} else if ( type.equals( "Double" ) ) {
valueMeta = new ValueMetaNumber();
} else if ( type.equals( "Long" ) ) {
valueMeta = new ValueMetaInteger();
} else if ( type.equals( "String" ) ) {
valueMeta = new ValueMetaString();
} else if ( type.equals( "Date" ) ) {
valueMeta = new ValueMetaDate();
} else if ( type.equals( "Boolean" ) ) {
valueMeta = new ValueMetaBoolean();
}
if ( valueMeta != null ) {
valueMeta.setName( name );
valueMeta.setComments( name );
valueMeta.setLength( size );
valueMeta.setOriginalColumnTypeName( type );
rowMeta.addValueMeta( valueMeta );
} else {
log.logBasic( "Database.getQueryFields() ValueMetaInterface mapping not resolved for the column " + name );
rowMeta = null;
break;
}
}
if ( rowMeta != null && !rowMeta.isEmpty() ) {
return rowMeta;
} else {
throw new Exception( "Error in Database.getQueryFields()" );
}
}

public RowMetaInterface getQueryFieldsFallback( String sql, boolean param, RowMetaInterface inform,
Object[] data ) throws KettleDatabaseException {
RowMetaInterface fields;

Expand Down
96 changes: 89 additions & 7 deletions core/test-src/org/pentaho/di/core/database/DatabaseUnitTest.java
View file
Expand Up @@ -38,13 +38,7 @@
import static org.mockito.Mockito.when;

import java.lang.reflect.Field;
import java.sql.BatchUpdateException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.*;
import java.util.List;

import javax.sql.DataSource;
Expand All @@ -62,6 +56,7 @@
import org.pentaho.di.core.logging.SimpleLoggingObject;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaNumber;
import org.pentaho.di.core.variables.VariableSpace;

public class DatabaseUnitTest {
Expand All @@ -73,6 +68,93 @@ public static void setUp() throws Exception {
KettleClientEnvironment.init();
}

@Test
public void testGetQueryFieldsFromPreparedStatement() throws Exception {
String sql = "select * from employees";
String columnName = "salary";

DatabaseMeta meta = Mockito.mock( DatabaseMeta.class );
PreparedStatement ps = Mockito.mock( PreparedStatement.class );
Connection conn = mockConnection( mock( DatabaseMetaData.class ) );
ResultSetMetaData rsMetaData = mock( ResultSetMetaData.class );

when( rsMetaData.getColumnCount() ).thenReturn( 1 );
when( rsMetaData.getColumnName( 1 ) ).thenReturn( columnName );
when( rsMetaData.getColumnLabel( 1 ) ).thenReturn( columnName );
when( rsMetaData.getColumnType( 1 ) ).thenReturn( Types.DECIMAL );

Mockito.when( meta.stripCR( anyString() ) ).thenReturn( sql );
Mockito.when( meta.getDatabaseInterface() ).thenReturn( new MySQLDatabaseMeta() );
Mockito.when( conn.prepareStatement( sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY ) )
.thenReturn( ps );
Mockito.when( ps.getMetaData() ).thenReturn( rsMetaData );

Database db = new Database( log, meta );
db.setConnection( conn );
RowMetaInterface rowMetaInterface = db.getQueryFieldsFromPreparedStatement( sql );

assertEquals( rowMetaInterface.size(), 1 );
assertEquals( rowMetaInterface.getValueMeta( 0 ).getName(), columnName );
assertTrue( rowMetaInterface.getValueMeta( 0 ) instanceof ValueMetaNumber );
}

@Test
public void testGetQueryFieldsFromDatabaseMetaData() throws Exception {
DatabaseMeta meta = Mockito.mock( DatabaseMeta.class );
DatabaseMetaData dbMetaData = mock( DatabaseMetaData.class );
Connection conn = mockConnection( dbMetaData );
ResultSet columns = mock( ResultSet.class );
String columnName = "year";
String columnType = "Integer";
int columnSize = 15;

Mockito.when( dbMetaData.getColumns( anyString(), anyString(), anyString(), anyString() ) ).thenReturn( columns );
Mockito.when( columns.next() ).thenReturn( true ).thenReturn( false );
Mockito.when( columns.getString( "COLUMN_NAME" ) ).thenReturn( columnName );
Mockito.when( columns.getString( "SOURCE_DATA_TYPE" ) ).thenReturn( columnType );
Mockito.when( columns.getInt( "COLUMN_SIZE" ) ).thenReturn( columnSize );

Database db = new Database( log, meta );
db.setConnection( conn );
RowMetaInterface rowMetaInterface = db.getQueryFieldsFromDatabaseMetaData();

assertEquals( rowMetaInterface.size(), 1 );
assertEquals( rowMetaInterface.getValueMeta( 0 ).getName(), columnName );
assertEquals( rowMetaInterface.getValueMeta( 0 ).getOriginalColumnTypeName(), columnType );
assertEquals( rowMetaInterface.getValueMeta( 0 ).getLength(), columnSize );
}

@Test
public void testGetQueryFieldsFallback() throws Exception {
String sql = "select * from employees";
String columnName = "salary";

DatabaseMeta meta = Mockito.mock( DatabaseMeta.class );
PreparedStatement ps = Mockito.mock( PreparedStatement.class );
Connection conn = mockConnection( mock( DatabaseMetaData.class ) );
ResultSetMetaData rsMetaData = mock( ResultSetMetaData.class );
ResultSet rs = Mockito.mock( ResultSet.class );

when( rsMetaData.getColumnCount() ).thenReturn( 1 );
when( rsMetaData.getColumnName( 1 ) ).thenReturn( columnName );
when( rsMetaData.getColumnLabel( 1 ) ).thenReturn( columnName );
when( rsMetaData.getColumnType( 1 ) ).thenReturn( Types.DECIMAL );
when( ps.executeQuery() ).thenReturn( rs );

Mockito.when( meta.stripCR( anyString() ) ).thenReturn( sql );
Mockito.when( meta.getDatabaseInterface() ).thenReturn( new MySQLDatabaseMeta() );
Mockito.when( conn.prepareStatement( sql ) ).thenReturn( ps );
Mockito.when( rs.getMetaData() ).thenReturn( rsMetaData );

Database db = new Database( log, meta );
db.setConnection( conn );
RowMetaInterface rowMetaInterface = db.getQueryFieldsFallback( sql, false, null, null );

assertEquals( rowMetaInterface.size(), 1 );
assertEquals( rowMetaInterface.getValueMeta( 0 ).getName(), columnName );
assertTrue( rowMetaInterface.getValueMeta( 0 ) instanceof ValueMetaNumber );
}

/**
* PDI-11363. when using getLookup calls there is no need to make attempt to retrieve row set metadata for every call.
* That may bring performance penalty depends on jdbc driver implementation. For some drivers that penalty can be huge
Expand Down

0 comments on commit 4ebd088

Please sign in to comment.