Skip to content

Commit

Permalink
Revert "[BACKLOG-16814] Base classes for input steps (#4033)" (#4035)
Browse files Browse the repository at this point in the history 
This reverts commit dd82086.
  • Loading branch information
Ben Morrise committed Jun 19, 2017
1 parent dd82086 commit a853d42
Show file tree
Hide file tree
Showing 14 changed files with 78 additions and 266 deletions.
6 changes: 3 additions & 3 deletions engine/src/org/pentaho/di/trans/steps/fileinput/BaseFileInputStep.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -244,8 +244,8 @@ protected void prepareToRowProcessing() throws KettleException {
data.dataErrorLineHandler );

// Count the number of repeat fields...
for ( int i = 0; i < meta.inputFields.length; i++ ) {
if ( meta.inputFields[i].isRepeated() ) {
for ( int i = 0; i < meta.inputFiles.inputFields.length; i++ ) {
if ( meta.inputFiles.inputFields[i].isRepeated() ) {
data.nr_repeats++;
}
}
Expand Down
18 changes: 8 additions & 10 deletions engine/src/org/pentaho/di/trans/steps/fileinput/BaseFileInputStepMeta.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -37,15 +37,14 @@
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStepMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

/**
* Base meta for file-based input steps.
*
* @author Alexander Buloichik
*/
public abstract class BaseFileInputStepMeta<A extends BaseFileInputStepMeta.AdditionalOutputFields, I extends BaseFileInputStepMeta.InputFiles, F extends BaseFileInputField>
extends BaseStepMeta implements StepMetaInterface {
public abstract class BaseFileInputStepMeta<A extends BaseFileInputStepMeta.AdditionalOutputFields, I extends BaseFileInputStepMeta.InputFiles<? extends BaseFileInputField>>
extends BaseStepMeta {
private static Class<?> PKG = BaseFileInputStepMeta.class; // for i18n purposes, needed by Translator2!!

public static final String[] RequiredFilesCode = new String[] { "N", "Y" };
Expand All @@ -60,11 +59,6 @@ public abstract class BaseFileInputStepMeta<A extends BaseFileInputStepMeta.Addi

@InjectionDeep
public I inputFiles;

/** The fields to import... */
@InjectionDeep
public F[] inputFields;

@InjectionDeep
public ErrorHandling errorHandling = new ErrorHandling();
@InjectionDeep
Expand All @@ -73,7 +67,7 @@ public abstract class BaseFileInputStepMeta<A extends BaseFileInputStepMeta.Addi
/**
* Input files settings.
*/
public static class InputFiles implements Cloneable {
public static class InputFiles<F extends BaseFileInputField> implements Cloneable {

/** Array of filenames */
@Injection( name = "FILENAME", group = "FILENAME_LINES" )
Expand Down Expand Up @@ -111,6 +105,10 @@ public static class InputFiles implements Cloneable {
@Injection( name = "ACCEPT_FILE_FIELD" )
public String acceptingField;

/** The fields to import... */
@InjectionDeep
public F[] inputFields;

/** The add filenames to result filenames flag */
@Injection( name = "ADD_FILES_TO_RESULT" )
public boolean isaddresult;
Expand Down
30 changes: 15 additions & 15 deletions engine/src/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMeta.java
View file
Expand Up @@ -73,7 +73,7 @@

@InjectionSupported( localizationPrefix = "TextFileInput.Injection.", groups = { "FILENAME_LINES", "FIELDS", "FILTERS" } )
public class TextFileInputMeta extends
BaseFileInputStepMeta<BaseFileInputStepMeta.AdditionalOutputFields, BaseFileInputStepMeta.InputFiles, BaseFileInputField>
BaseFileInputStepMeta<BaseFileInputStepMeta.AdditionalOutputFields, BaseFileInputStepMeta.InputFiles<BaseFileInputField>>
implements StepMetaInterface {
private static Class<?> PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! TODO: check i18n
// for base
Expand Down Expand Up @@ -231,8 +231,8 @@ public void setDateFormatLocale( String locale ) {

public TextFileInputMeta() {
additionalOutputFields = new BaseFileInputStepMeta.AdditionalOutputFields();
inputFiles = new BaseFileInputStepMeta.InputFiles();
inputFields = new BaseFileInputField[0];
inputFiles = new BaseFileInputStepMeta.InputFiles<>();
inputFiles.inputFields = new BaseFileInputField[0];
}

/**
Expand Down Expand Up @@ -384,7 +384,7 @@ public void loadXML( Node stepnode, List<DatabaseMeta> databases, IMetaStore met
field.setTrimType( ValueMetaString.getTrimTypeByCode( XMLHandler.getTagValue( fnode, "trim_type" ) ) );
field.setRepeated( YES.equalsIgnoreCase( XMLHandler.getTagValue( fnode, "repeat" ) ) );

inputFields[i] = field;
inputFiles.inputFields[i] = field;
}

// Is there a limit on the number of rows we process?
Expand Down Expand Up @@ -436,7 +436,7 @@ public Object clone() {
TextFileInputMeta retval = (TextFileInputMeta) super.clone();

int nrfiles = inputFiles.fileName.length;
int nrfields = inputFields.length;
int nrfields = inputFiles.inputFields.length;
int nrfilters = filter.length;

retval.allocate( nrfiles, nrfields, nrfilters );
Expand All @@ -448,7 +448,7 @@ public Object clone() {
System.arraycopy( inputFiles.includeSubFolders, 0, retval.inputFiles.includeSubFolders, 0, nrfiles );

for ( int i = 0; i < nrfields; i++ ) {
retval.inputFields[i] = (BaseFileInputField) inputFields[i].clone();
retval.inputFiles.inputFields[i] = (BaseFileInputField) inputFiles.inputFields[i].clone();
}

for ( int i = 0; i < nrfilters; i++ ) {
Expand All @@ -461,7 +461,7 @@ public Object clone() {
public void allocate( int nrfiles, int nrfields, int nrfilters ) {
allocateFiles( nrfiles );

inputFields = new BaseFileInputField[nrfields];
inputFiles.inputFields = new BaseFileInputField[nrfields];
filter = new TextFileFilter[nrfilters];
}

Expand Down Expand Up @@ -534,7 +534,7 @@ public void setDefault() {
}

for ( int i = 0; i < nrfields; i++ ) {
inputFields[i] = new BaseFileInputField( "field" + ( i + 1 ), 1, -1 );
inputFiles.inputFields[i] = new BaseFileInputField( "field" + ( i + 1 ), 1, -1 );
}

content.dateFormatLocale = Locale.getDefault();
Expand All @@ -560,8 +560,8 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf
}
}

for ( int i = 0; i < inputFields.length; i++ ) {
BaseFileInputField field = inputFields[i];
for ( int i = 0; i < inputFiles.inputFields.length; i++ ) {
BaseFileInputField field = inputFiles.inputFields[i];

int type = field.getType();
if ( type == ValueMetaInterface.TYPE_NONE ) {
Expand Down Expand Up @@ -756,8 +756,8 @@ public String getXML() {
retval.append( " </filters>" ).append( Const.CR );

retval.append( " <fields>" ).append( Const.CR );
for ( int i = 0; i < inputFields.length; i++ ) {
BaseFileInputField field = inputFields[i];
for ( int i = 0; i < inputFiles.inputFields.length; i++ ) {
BaseFileInputField field = inputFiles.inputFields[i];

retval.append( " <field>" ).append( Const.CR );
retval.append( " " ).append( XMLHandler.addTagValue( "name", field.getName() ) );
Expand Down Expand Up @@ -940,7 +940,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis
field.setTrimType( ValueMetaString.getTrimTypeByCode( rep.getStepAttributeString( id_step, i, "field_trim_type" ) ) );
field.setRepeated( rep.getStepAttributeBoolean( id_step, i, "field_repeat" ) );

inputFields[i] = field;
inputFiles.inputFields[i] = field;
}

errorHandling.errorIgnored = rep.getStepAttributeBoolean( id_step, "error_ignored" );
Expand Down Expand Up @@ -1040,8 +1040,8 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform
rep.saveStepAttribute( id_transformation, id_step, i, "filter_is_positive", filter[i].isFilterPositive() );
}

for ( int i = 0; i < inputFields.length; i++ ) {
BaseFileInputField field = inputFields[i];
for ( int i = 0; i < inputFiles.inputFields.length; i++ ) {
BaseFileInputField field = inputFiles.inputFields[i];

rep.saveStepAttribute( id_transformation, id_step, i, "field_name", field.getName() );
rep.saveStepAttribute( id_transformation, id_step, i, "field_type", field.getTypeDesc() );
Expand Down
4 changes: 2 additions & 2 deletions engine/src/org/pentaho/di/trans/steps/fileinput/text/TextFileInputReader.java
View file
Expand Up @@ -333,8 +333,8 @@ public boolean readRow() throws KettleException {
data.previous_row = data.outputRowMeta.cloneRow( r );
} else {
// int repnr = 0;
for ( int i = 0; i < meta.inputFields.length; i++ ) {
if ( meta.inputFields[i].isRepeated() ) {
for ( int i = 0; i < meta.inputFiles.inputFields.length; i++ ) {
if ( meta.inputFiles.inputFields[i].isRepeated() ) {
if ( r[i] == null ) {
// if it is empty: take the previous value!

Expand Down
16 changes: 8 additions & 8 deletions engine/src/org/pentaho/di/trans/steps/fileinput/text/TextFileInputUtils.java
View file
Expand Up @@ -234,8 +234,8 @@ public static final String[] guessStringsFromLine( VariableSpace space, LogChann
}
} else {
// Fixed file format: Simply get the strings at the required positions...
for ( int i = 0; i < inf.inputFields.length; i++ ) {
BaseFileInputField field = inf.inputFields[i];
for ( int i = 0; i < inf.inputFiles.inputFields.length; i++ ) {
BaseFileInputField field = inf.inputFiles.inputFields[i];

int length = line.length();

Expand Down Expand Up @@ -349,7 +349,7 @@ public static final Object[] convertLineToRow( LogChannelInterface log, TextFile
Object[] r = RowDataUtil.allocateRowData( outputRowMeta.size() ); // over-allocate a bit in the row producing
// steps...

int nrfields = info.inputFields.length;
int nrfields = info.inputFiles.inputFields.length;
int fieldnr;

Long errorCount = null;
Expand All @@ -373,7 +373,7 @@ public static final Object[] convertLineToRow( LogChannelInterface log, TextFile
String[] strings = convertLineToStrings( log, textFileLine.line, info, delimiter, enclosure, escapeCharacter );
int shiftFields = ( passThruFields == null ? 0 : nrPassThruFields );
for ( fieldnr = 0; fieldnr < nrfields; fieldnr++ ) {
BaseFileInputField f = info.inputFields[fieldnr];
BaseFileInputField f = info.inputFiles.inputFields[fieldnr];
int valuenr = shiftFields + fieldnr;
ValueMetaInterface valueMeta = outputRowMeta.getValueMeta( valuenr );
ValueMetaInterface convertMeta = convertRowMeta.getValueMeta( valuenr );
Expand Down Expand Up @@ -449,7 +449,7 @@ public static final Object[] convertLineToRow( LogChannelInterface log, TextFile
// Support for trailing nullcols!
// Should be OK at allocation time, but it doesn't hurt :-)
if ( fieldnr < nrfields ) {
for ( int i = fieldnr; i < info.inputFields.length; i++ ) {
for ( int i = fieldnr; i < info.inputFiles.inputFields.length; i++ ) {
r[shiftFields + i] = null;
}
}
Expand Down Expand Up @@ -538,7 +538,7 @@ public static final Object[] convertLineToRow( LogChannelInterface log, TextFile

public static final String[] convertLineToStrings( LogChannelInterface log, String line, TextFileInputMeta inf,
String delimiter, String enclosure, String escapeCharacters ) throws KettleException {
String[] strings = new String[inf.inputFields.length];
String[] strings = new String[inf.inputFiles.inputFields.length];
int fieldnr;

String pol; // piece of line
Expand Down Expand Up @@ -742,8 +742,8 @@ public static final String[] convertLineToStrings( LogChannelInterface log, Stri
// it will still use the old behavior. The *only* way to get the new behavior is if content.length = "Bytes" and
// the encoding is specified.
boolean charBased = ( inf.content.length == null || inf.content.length.equalsIgnoreCase( "Characters" ) || inf.getEncoding() == null ); // Default to classic behavior
for ( int i = 0; i < inf.inputFields.length; i++ ) {
BaseFileInputField field = inf.inputFields[i];
for ( int i = 0; i < inf.inputFiles.inputFields.length; i++ ) {
BaseFileInputField field = inf.inputFiles.inputFields[i];

int length;
int fPos = field.getPosition();
Expand Down
4 changes: 2 additions & 2 deletions engine/test-src/org/pentaho/di/trans/steps/fileinput/text/BaseTextParsingTest.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2016-2017 by Pentaho : http://www.pentaho.com
* Copyright (C) 2016 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -76,7 +76,7 @@ protected void initByURL( String url ) throws Exception {
* TODO: move to BaseParsingTest after CSV moving to BaseFileInput
*/
protected void setFields( BaseFileInputField... fields ) throws Exception {
meta.inputFields = fields;
meta.inputFiles.inputFields = fields;
meta.getFields( data.outputRowMeta, meta.getName(), null, null, new Variables(), null, null );
data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING );
}
Expand Down
28 changes: 14 additions & 14 deletions ...test-src/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMetaNewInjectionTest.java
View file
Expand Up @@ -289,19 +289,19 @@ public boolean get() {
/////////////////////////////
check( "FIELD_NAME", new StringGetter() {
public String get() {
return meta.inputFields[0].getName();
return meta.inputFiles.inputFields[0].getName();
}
} );

check( "FIELD_POSITION", new IntGetter() {
public int get() {
return meta.inputFields[0].getPosition();
return meta.inputFiles.inputFields[0].getPosition();
}
} );

check( "FIELD_LENGTH", new IntGetter() {
public int get() {
return meta.inputFields[0].getLength();
return meta.inputFiles.inputFields[0].getLength();
}
} );

Expand All @@ -314,62 +314,62 @@ public int get() {

check( "FIELD_IGNORE", new BooleanGetter() {
public boolean get() {
return meta.inputFields[0].isIgnored();
return meta.inputFiles.inputFields[0].isIgnored();
}
} );

check( "FIELD_FORMAT", new StringGetter() {
public String get() {
return meta.inputFields[0].getFormat();
return meta.inputFiles.inputFields[0].getFormat();
}
} );

ValueMetaInterface mftt = new ValueMetaString( "f" );
injector.setProperty( meta, "FIELD_TRIM_TYPE", setValue( mftt, "left" ), "f" );
assertEquals( 1, meta.inputFields[0].getTrimType() );
assertEquals( 1, meta.inputFiles.inputFields[0].getTrimType() );
injector.setProperty( meta, "FIELD_TRIM_TYPE", setValue( mftt, "right" ), "f" );
assertEquals( 2, meta.inputFields[0].getTrimType() );
assertEquals( 2, meta.inputFiles.inputFields[0].getTrimType() );
skipPropertyTest( "FIELD_TRIM_TYPE" );

check( "FIELD_PRECISION", new IntGetter() {
public int get() {
return meta.inputFields[0].getPrecision();
return meta.inputFiles.inputFields[0].getPrecision();
}
} );

check( "FIELD_CURRENCY", new StringGetter() {
public String get() {
return meta.inputFields[0].getCurrencySymbol();
return meta.inputFiles.inputFields[0].getCurrencySymbol();
}
} );

check( "FIELD_DECIMAL", new StringGetter() {
public String get() {
return meta.inputFields[0].getDecimalSymbol();
return meta.inputFiles.inputFields[0].getDecimalSymbol();
}
} );

check( "FIELD_GROUP", new StringGetter() {
public String get() {
return meta.inputFields[0].getGroupSymbol();
return meta.inputFiles.inputFields[0].getGroupSymbol();
}
} );

check( "FIELD_REPEAT", new BooleanGetter() {
public boolean get() {
return meta.inputFields[0].isRepeated();
return meta.inputFiles.inputFields[0].isRepeated();
}
} );

check( "FIELD_NULL_STRING", new StringGetter() {
public String get() {
return meta.inputFields[0].getNullString();
return meta.inputFiles.inputFields[0].getNullString();
}
} );

check( "FIELD_IF_NULL", new StringGetter() {
public String get() {
return meta.inputFields[0].getIfNullValue();
return meta.inputFiles.inputFields[0].getIfNullValue();
}
} );

Expand Down
4 changes: 2 additions & 2 deletions engine/test-src/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMetaTest.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -60,7 +60,7 @@ public void setUp() throws Exception {

@Test
public void whenExportingResourcesWeGetFileObjectsOnlyFromFilesWithNotNullAndNotEmptyFileNames() throws Exception {
inputMeta.inputFiles = new BaseFileInputStepMeta.InputFiles();
inputMeta.inputFiles = new BaseFileInputStepMeta.InputFiles<>();
inputMeta.inputFiles.fileName = new String[] { FILE_NAME_NULL, FILE_NAME_EMPTY, FILE_NAME_VALID_PATH };
inputMeta.inputFiles.fileMask =
new String[] { StringUtil.EMPTY_STRING, StringUtil.EMPTY_STRING, StringUtil.EMPTY_STRING };
Expand Down

0 comments on commit a853d42

Please sign in to comment.