Skip to content

Commit

Permalink
[PDI-14347] Use any ValueMeta with UDJE step
Browse files Browse the repository at this point in the history 
Remove ValueMeta-specific switch statement, and use ValueMetaFactory to
allow extensible ValueMetaInterface classes.
Perform validation to ensure return values still map to appropriate
ValueMeta classes
  • Loading branch information
Matt Tucker committed May 16, 2016
1 parent 57790c7 commit 7f1ca84
Show file tree
Hide file tree
Showing 8 changed files with 68 additions and 167 deletions.
153 changes: 16 additions & 137 deletions engine/src/org/pentaho/di/trans/steps/janino/Janino.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand All @@ -22,9 +22,7 @@


package org.pentaho.di.trans.steps.janino; package org.pentaho.di.trans.steps.janino;


import java.math.BigDecimal;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;


import org.codehaus.janino.ExpressionEvaluator; import org.codehaus.janino.ExpressionEvaluator;
Expand All @@ -34,6 +32,8 @@
import org.pentaho.di.core.row.RowDataUtil; import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaFactory;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep; import org.pentaho.di.trans.step.BaseStep;
Expand All @@ -49,6 +49,7 @@
* @since 8-sep-2005 * @since 8-sep-2005
*/ */
public class Janino extends BaseStep implements StepInterface { public class Janino extends BaseStep implements StepInterface {
private static Class<?> PKG = JaninoMeta.class;
private JaninoMeta meta; private JaninoMeta meta;
private JaninoData data; private JaninoData data;


Expand Down Expand Up @@ -77,8 +78,10 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
// Calculate replace indexes... // Calculate replace indexes...
// //
data.replaceIndex = new int[meta.getFormula().length]; data.replaceIndex = new int[meta.getFormula().length];
data.returnType = new ValueMetaInterface[meta.getFormula().length];
for ( int i = 0; i < meta.getFormula().length; i++ ) { for ( int i = 0; i < meta.getFormula().length; i++ ) {
JaninoMetaFunction fn = meta.getFormula()[i]; JaninoMetaFunction fn = meta.getFormula()[i];
data.returnType[i] = ValueMetaFactory.createValueMeta( fn.getValueType() );
if ( !Const.isEmpty( fn.getReplaceField() ) ) { if ( !Const.isEmpty( fn.getReplaceField() ) ) {
data.replaceIndex[i] = getInputRowMeta().indexOfValue( fn.getReplaceField() ); data.replaceIndex[i] = getInputRowMeta().indexOfValue( fn.getReplaceField() );
if ( data.replaceIndex[i] < 0 ) { if ( data.replaceIndex[i] < 0 ) {
Expand Down Expand Up @@ -147,34 +150,7 @@ private Object[] calcFields( RowMetaInterface rowMeta, Object[] r ) throws Kettl
// If so, add it to the indexes... // If so, add it to the indexes...
argIndexes.add( i ); argIndexes.add( i );


Class<?> parameterType; parameterTypes.add( valueMeta.getNativeDataTypeClass() );
switch ( valueMeta.getType() ) {
case ValueMetaInterface.TYPE_STRING:
parameterType = String.class;
break;
case ValueMetaInterface.TYPE_NUMBER:
parameterType = Double.class;
break;
case ValueMetaInterface.TYPE_INTEGER:
parameterType = Long.class;
break;
case ValueMetaInterface.TYPE_DATE:
parameterType = Date.class;
break;
case ValueMetaInterface.TYPE_BIGNUMBER:
parameterType = BigDecimal.class;
break;
case ValueMetaInterface.TYPE_BOOLEAN:
parameterType = Boolean.class;
break;
case ValueMetaInterface.TYPE_BINARY:
parameterType = byte[].class;
break;
default:
parameterType = String.class;
break;
}
parameterTypes.add( parameterType );
parameterNames.add( valueMeta.getName() ); parameterNames.add( valueMeta.getName() );
} }
} }
Expand All @@ -199,8 +175,6 @@ private Object[] calcFields( RowMetaInterface rowMeta, Object[] r ) throws Kettl
} }


for ( int i = 0; i < meta.getFormula().length; i++ ) { for ( int i = 0; i < meta.getFormula().length; i++ ) {
JaninoMetaFunction fn = meta.getFormula()[i];

List<Integer> argumentIndexes = data.argumentIndexes.get( i ); List<Integer> argumentIndexes = data.argumentIndexes.get( i );


// This method can only accept the specified number of values... // This method can only accept the specified number of values...
Expand All @@ -211,104 +185,20 @@ private Object[] calcFields( RowMetaInterface rowMeta, Object[] r ) throws Kettl
ValueMetaInterface outputValueMeta = data.outputRowMeta.getValueMeta( index ); ValueMetaInterface outputValueMeta = data.outputRowMeta.getValueMeta( index );
argumentData[x] = outputValueMeta.convertToNormalStorageType( outputRowData[index] ); argumentData[x] = outputValueMeta.convertToNormalStorageType( outputRowData[index] );
} }
// System.arraycopy(outputRowData, 0, argumentData, 0, argumentData.length);


Object formulaResult = data.expressionEvaluators[i].evaluate( argumentData ); Object formulaResult = data.expressionEvaluators[i].evaluate( argumentData );


// Calculate the return type on the first row... Object value = null;
//
if ( data.returnType[i] < 0 ) {
if ( formulaResult instanceof String ) {
data.returnType[i] = JaninoData.RETURN_TYPE_STRING;
if ( fn.getValueType() != ValueMetaInterface.TYPE_STRING ) {
throw new KettleValueException( "Please specify a String type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof Integer ) {
data.returnType[i] = JaninoData.RETURN_TYPE_INTEGER;
if ( fn.getValueType() != ValueMetaInterface.TYPE_INTEGER ) {
throw new KettleValueException( "Please specify an Integer type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof Long ) {
data.returnType[i] = JaninoData.RETURN_TYPE_LONG;
if ( fn.getValueType() != ValueMetaInterface.TYPE_INTEGER ) {
throw new KettleValueException( "Please specify an Integer type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof BigDecimal ) { // BigDecimal must be before Number since this is also
// instanceof Number
data.returnType[i] = JaninoData.RETURN_TYPE_BIGDECIMAL;
if ( fn.getValueType() != ValueMetaInterface.TYPE_BIGNUMBER ) {
throw new KettleValueException( "Please specify a BigNumber type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof Number ) {
data.returnType[i] = JaninoData.RETURN_TYPE_NUMBER;
if ( fn.getValueType() != ValueMetaInterface.TYPE_NUMBER ) {
throw new KettleValueException( "Please specify a Number type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof Date ) {
data.returnType[i] = JaninoData.RETURN_TYPE_DATE;
if ( fn.getValueType() != ValueMetaInterface.TYPE_DATE ) {
throw new KettleValueException( "Please specify a Date type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof byte[] ) {
data.returnType[i] = JaninoData.RETURN_TYPE_BYTE_ARRAY;
if ( fn.getValueType() != ValueMetaInterface.TYPE_BINARY ) {
throw new KettleValueException( "Please specify a Binary type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
} else if ( formulaResult instanceof Boolean ) {
data.returnType[i] = JaninoData.RETURN_TYPE_BOOLEAN;
if ( fn.getValueType() != ValueMetaInterface.TYPE_BOOLEAN ) {
throw new KettleValueException( "Please specify a Boolean type to parse ["
+ formulaResult.getClass().getName() + "] for field [" + fn.getFieldName()
+ "] as a result of formula [" + fn.getFormula() + "]" );
}
}
}

Object value;
if ( formulaResult == null ) { if ( formulaResult == null ) {
value = null; value = null;
} else { } else {
switch ( data.returnType[i] ) { ValueMetaInterface valueMeta = data.returnType[i];
case JaninoData.RETURN_TYPE_STRING: if ( valueMeta.getNativeDataTypeClass().isAssignableFrom( formulaResult.getClass() ) ) {
value = formulaResult.toString(); value = formulaResult;
break; } else {
case JaninoData.RETURN_TYPE_NUMBER: throw new KettleValueException(
value = new Double( ( (Number) formulaResult ).doubleValue() ); BaseMessages.getString( PKG, "Janino.Error.ValueTypeMismatch", valueMeta.getTypeDesc(),
break; meta.getFormula()[i].getFieldName(), formulaResult.getClass(), meta.getFormula()[i].getFormula() ) );
case JaninoData.RETURN_TYPE_INTEGER:
value = new Long( ( (Integer) formulaResult ).intValue() );
break;
case JaninoData.RETURN_TYPE_LONG:
value = formulaResult;
break;
case JaninoData.RETURN_TYPE_DATE:
value = formulaResult;
break;
case JaninoData.RETURN_TYPE_BIGDECIMAL:
value = formulaResult;
break;
case JaninoData.RETURN_TYPE_BYTE_ARRAY:
value = formulaResult;
break;
case JaninoData.RETURN_TYPE_BOOLEAN:
value = formulaResult;
break;
default:
value = null;
} }
} }


Expand All @@ -331,18 +221,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {
meta = (JaninoMeta) smi; meta = (JaninoMeta) smi;
data = (JaninoData) sdi; data = (JaninoData) sdi;


if ( super.init( smi, sdi ) ) { return super.init( smi, sdi );
// Add init code here.

// Return data type discovery is expensive, let's discover them one time only.
//
data.returnType = new int[meta.getFormula().length];
for ( int i = 0; i < meta.getFormula().length; i++ ) {
data.returnType[i] = -1;
}
return true;
}
return false;
} }


} }
5 changes: 3 additions & 2 deletions engine/src/org/pentaho/di/trans/steps/janino/JaninoData.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand All @@ -26,6 +26,7 @@


import org.codehaus.janino.ExpressionEvaluator; import org.codehaus.janino.ExpressionEvaluator;
import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.step.BaseStepData; import org.pentaho.di.trans.step.BaseStepData;
import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepDataInterface;


Expand All @@ -45,7 +46,7 @@ public class JaninoData extends BaseStepData implements StepDataInterface {
public static final int RETURN_TYPE_BOOLEAN = 7; public static final int RETURN_TYPE_BOOLEAN = 7;


public RowMetaInterface outputRowMeta; public RowMetaInterface outputRowMeta;
public int[] returnType; public ValueMetaInterface[] returnType;
public int[] replaceIndex; public int[] replaceIndex;


public ExpressionEvaluator[] expressionEvaluators; public ExpressionEvaluator[] expressionEvaluators;
Expand Down
2 changes: 1 addition & 1 deletion engine/src/org/pentaho/di/trans/steps/janino/JaninoMeta.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand Down
2 changes: 1 addition & 1 deletion engine/src/org/pentaho/di/trans/steps/janino/JaninoMetaFunction.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand Down
11 changes: 5 additions & 6 deletions engine/src/org/pentaho/di/trans/steps/janino/JaninoMetaInjection.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand All @@ -24,8 +24,8 @@


import org.pentaho.di.core.Const; import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaFactory;
import org.pentaho.di.trans.step.StepInjectionMetaEntry; import org.pentaho.di.trans.step.StepInjectionMetaEntry;
import org.pentaho.di.trans.step.StepInjectionUtil; import org.pentaho.di.trans.step.StepInjectionUtil;
import org.pentaho.di.trans.step.StepMetaInjectionEntryInterface; import org.pentaho.di.trans.step.StepMetaInjectionEntryInterface;
Expand Down Expand Up @@ -130,7 +130,6 @@ public void injectStepMetadataEntries( List<StepInjectionMetaEntry> all ) throws
continue; continue;
} }


String lookValue = (String) lookFields.getValue();
switch ( fieldsEntry ) { switch ( fieldsEntry ) {
case EXPRESSION_FIELDS: case EXPRESSION_FIELDS:
for ( StepInjectionMetaEntry lookField : lookFields.getDetails() ) { for ( StepInjectionMetaEntry lookField : lookFields.getDetails() ) {
Expand Down Expand Up @@ -204,8 +203,8 @@ public void injectStepMetadataEntries( List<StepInjectionMetaEntry> all ) throws


while ( iFieldNames.hasNext() ) { while ( iFieldNames.hasNext() ) {
fields[i] = new JaninoMetaFunction( iFieldNames.next(), iJavaExpressions.next(), fields[i] = new JaninoMetaFunction( iFieldNames.next(), iJavaExpressions.next(),
ValueMeta.getType( iValueTypes.next() ), Const.toInt( iLengths.next(), -1 ), ValueMetaFactory.getIdForValueMeta( iValueTypes.next() ), Const.toInt( iLengths.next(), -1 ),
Const.toInt( iPrecisions.next(), -1 ), iReplaceValues.next() ); Const.toInt( iPrecisions.next(), -1 ), iReplaceValues.next() );


i++; i++;
} }
Expand All @@ -224,7 +223,7 @@ public List<StepInjectionMetaEntry> extractStepMetadataEntries() throws KettleEx
List<StepInjectionMetaEntry> details = fieldEntry.getDetails(); List<StepInjectionMetaEntry> details = fieldEntry.getDetails();
details.add( StepInjectionUtil.getEntry( Entry.NEW_FIELDNAME, meta.getFormula()[i].getFieldName() ) ); details.add( StepInjectionUtil.getEntry( Entry.NEW_FIELDNAME, meta.getFormula()[i].getFieldName() ) );
details.add( StepInjectionUtil.getEntry( Entry.JAVA_EXPRESSION, meta.getFormula()[i].getFormula() ) ); details.add( StepInjectionUtil.getEntry( Entry.JAVA_EXPRESSION, meta.getFormula()[i].getFormula() ) );
details.add( StepInjectionUtil.getEntry( Entry.VALUE_TYPE, ValueMeta.getTypeDesc( meta.getFormula()[i].getValueType() ) ) ); details.add( StepInjectionUtil.getEntry( Entry.VALUE_TYPE, ValueMetaFactory.getValueMetaName( meta.getFormula()[i].getValueType() ) ) );
details.add( StepInjectionUtil.getEntry( Entry.LENGTH, meta.getFormula()[i].getValueLength() ) ); details.add( StepInjectionUtil.getEntry( Entry.LENGTH, meta.getFormula()[i].getValueLength() ) );
details.add( StepInjectionUtil.getEntry( Entry.PRECISION, meta.getFormula()[i].getValuePrecision() ) ); details.add( StepInjectionUtil.getEntry( Entry.PRECISION, meta.getFormula()[i].getValuePrecision() ) );
details.add( StepInjectionUtil.getEntry( Entry.REPLACE_VALUE, meta.getFormula()[i].getReplaceField() ) ); details.add( StepInjectionUtil.getEntry( Entry.REPLACE_VALUE, meta.getFormula()[i].getReplaceField() ) );
Expand Down
1 change: 1 addition & 0 deletions engine/src/org/pentaho/di/trans/steps/janino/messages/messages_en_US.properties
View file
Expand Up @@ -17,3 +17,4 @@ JaninoMeta.CheckResult.FieldsReceived = Step is connected to previous one, recei
JaninoMeta.CheckResult.ExpectedInputOk = Step is receiving info from other steps. JaninoMeta.CheckResult.ExpectedInputOk = Step is receiving info from other steps.
JaninoMeta.CheckResult.ExpectedInputError = No input received from other steps\! JaninoMeta.CheckResult.ExpectedInputError = No input received from other steps\!
JaninoDialog.Replace.Column = Replace value JaninoDialog.Replace.Column = Replace value
Janino.Error.ValueTypeMismatch=A {0} type was specified for field [{1}], but a [{2}] type was returned as a result of formula [{3}]
5 changes: 3 additions & 2 deletions test/org/pentaho/di/trans/steps/janino/JaninoMetaInjectionTest.java
View file
Expand Up @@ -23,7 +23,8 @@
package org.pentaho.di.trans.steps.janino; package org.pentaho.di.trans.steps.janino;


import junit.framework.TestCase; import junit.framework.TestCase;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.value.ValueMetaFactory;
import org.pentaho.di.trans.step.StepInjectionMetaEntry; import org.pentaho.di.trans.step.StepInjectionMetaEntry;
import org.pentaho.di.trans.step.StepInjectionUtil; import org.pentaho.di.trans.step.StepInjectionUtil;


Expand Down Expand Up @@ -94,7 +95,7 @@ private JaninoMeta populateJaninoMeta() {
// CHECKSTYLE:Indentation:OFF // CHECKSTYLE:Indentation:OFF
for ( int i = 0; i < NR_FIELDS; i++ ) { for ( int i = 0; i < NR_FIELDS; i++ ) {
meta.getFormula()[i] = new JaninoMetaFunction( NEW_FIELDNAME + i, JAVA_EXPRESSION + i, meta.getFormula()[i] = new JaninoMetaFunction( NEW_FIELDNAME + i, JAVA_EXPRESSION + i,
ValueMeta.getType( VALUE_TYPE ), LENGTH + i, PRECISION + i, REPLACE_VALUE + i ); ValueMetaFactory.getIdForValueMeta( VALUE_TYPE ), LENGTH + i, PRECISION + i, REPLACE_VALUE + i );
} }


return meta; return meta;
Expand Down

0 comments on commit 7f1ca84

Please sign in to comment.