From 8b87341ea0a40957f7fe85c602a1c2ee64b000fb Mon Sep 17 00:00:00 2001 From: Alexander Buloichik Date: Tue, 25 Aug 2015 13:38:55 +0300 Subject: [PATCH] [BACKLOG-3705] Text File Input refactoring --- engine/src/kettle-steps.xml | 1 + .../step/messages/messages_en_US.properties | 1 + .../trans/steps/baseinput/BaseInputStep.java | 437 +++ .../steps/baseinput/BaseInputStepData.java | 80 + .../steps/baseinput/BaseInputStepMeta.java | 282 ++ .../trans/steps/baseinput/BaseStepUtils.java | 97 + .../steps/baseinput/IBaseInputReader.java | 34 + .../baseinput/IBaseInputStepControl.java | 48 + .../oldtextfileinput/OldEncodingType.java | 124 + .../oldtextfileinput/OldTextFileFilter.java | 128 + .../OldTextFileFilterProcessor.java | 119 + .../oldtextfileinput/OldTextFileInput.java | 1631 ++++++++ .../OldTextFileInputData.java | 164 + .../OldTextFileInputMeta.java | 2068 +++++++++++ .../OldTextFileInputMetaInjection.java | 688 ++++ .../oldtextfileinput/OldTextFileLine.java | 40 + .../steps/textfileinput/TextFileInput.java | 1604 +------- .../textfileinput/TextFileInputData.java | 83 +- .../textfileinput/TextFileInputField.java | 1 - .../textfileinput/TextFileInputMeta.java | 1685 +++------ .../TextFileInputMetaInjection.java | 219 +- .../textfileinput/TextFileInputReader.java | 466 +++ .../textfileinput/TextFileInputUtils.java | 758 ++++ .../oldtextfileinput/EncodingTypeTest.java | 63 + .../OldTextFileInputMetaInjectionTest.java | 184 + .../OldTextFileInputMetaLoadSaveTest.java | 210 ++ .../OldTextFileInputTest.java | 189 + .../steps/oldtextfileinput/PDI_2875_Test.java | 81 + .../steps/textfileinput/PDI_2875_Test.java | 12 +- .../TextFileInputContentParsingTest.java | 184 + .../TextFileInputMetaInjectionTest.java | 20 +- .../textfileinput/TextFileInputTest.java | 75 +- .../steps/textfileinput/texts/.gitattributes | 1 + .../steps/textfileinput/texts/default.csv | 4 + .../steps/s3csvinput/S3CsvInputDialog.java | 6 +- .../di/ui/core/widget/OldTableDraw.java | 541 +++ .../trans/steps/csvinput/CsvInputDialog.java | 12 +- ...dDirectoryDialogButtonListenerFactory.java | 51 + .../OldTextFileCSVImportProgressDialog.java | 498 +++ .../OldTextFileImportWizard.java | 154 + .../OldTextFileImportWizardPage1.java | 121 + .../OldTextFileImportWizardPage2.java | 712 ++++ .../OldTextFileInputDialog.java | 3263 +++++++++++++++++ .../OldVariableButtonListenerFactory.java | 91 + .../ParGzipCsvInputDialog.java | 8 +- .../TextFileCSVImportProgressDialog.java | 49 +- .../TextFileImportWizardPage1.java | 1 + .../TextFileImportWizardPage2.java | 3 + .../textfileinput/TextFileInputDialog.java | 353 +- 49 files changed, 14362 insertions(+), 3282 deletions(-) create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStep.java create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepData.java create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepMeta.java create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/BaseStepUtils.java create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputReader.java create mode 100644 engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputStepControl.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldEncodingType.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilter.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilterProcessor.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInput.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputData.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMeta.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjection.java create mode 100644 engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileLine.java create mode 100644 engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputReader.java create mode 100644 engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputUtils.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/EncodingTypeTest.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjectionTest.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaLoadSaveTest.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputTest.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/PDI_2875_Test.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputContentParsingTest.java create mode 100644 engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/.gitattributes create mode 100644 engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/default.csv create mode 100644 ui/src/org/pentaho/di/ui/core/widget/OldTableDraw.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldDirectoryDialogButtonListenerFactory.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileCSVImportProgressDialog.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizard.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage1.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage2.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileInputDialog.java create mode 100644 ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldVariableButtonListenerFactory.java diff --git a/engine/src/kettle-steps.xml b/engine/src/kettle-steps.xml index bf4b0a1ddfd2..0a49a630cef1 100644 --- a/engine/src/kettle-steps.xml +++ b/engine/src/kettle-steps.xml @@ -7,6 +7,7 @@ i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.SocketWriter org.pentaho.di.trans.steps.socketwriter.SocketWriterMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Inline i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.SocketWriter ui/images/SKW.svg http://wiki.pentaho.com/display/EAI/Socket+writer i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.SelectValues org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Transform i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.SelectValues ui/images/SEL.svg http://wiki.pentaho.com/display/EAI/Select+Values http://jira.pentaho.com/browse/PDI http://forums.pentaho.com/forumdisplay.php?135-Pentaho-Data-Integration-Kettle i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.TextFileInput org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Input i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.TextInputFile ui/images/TFI.svg http://wiki.pentaho.com/display/EAI/Text+File+Input http://jira.pentaho.com/browse/PDI http://forums.pentaho.com/forumdisplay.php?135-Pentaho-Data-Integration-Kettle + i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.OldTextFileInput org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileInputMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Input i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.OldTextInputFile ui/images/TFI.svg http://wiki.pentaho.com/display/EAI/Text+File+Input http://jira.pentaho.com/browse/PDI http://forums.pentaho.com/forumdisplay.php?135-Pentaho-Data-Integration-Kettle i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.Caculator org.pentaho.di.trans.steps.calculator.CalculatorMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Transform i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.Calculator ui/images/CLC.svg http://wiki.pentaho.com/display/EAI/Calculator i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.AddConstants org.pentaho.di.trans.steps.constant.ConstantMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Transform i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.Addconstants ui/images/CST.svg http://wiki.pentaho.com/display/EAI/Add+Constants i18n:org.pentaho.di.trans.step:BaseStep.TypeLongDesc.Abort org.pentaho.di.trans.steps.abort.AbortMeta i18n:org.pentaho.di.trans.step:BaseStep.Category.Flow i18n:org.pentaho.di.trans.step:BaseStep.TypeTooltipDesc.Abort ui/images/ABR.svg http://wiki.pentaho.com/display/EAI/Abort diff --git a/engine/src/org/pentaho/di/trans/step/messages/messages_en_US.properties b/engine/src/org/pentaho/di/trans/step/messages/messages_en_US.properties index 8aa66a2f04a9..f4571ee61afc 100644 --- a/engine/src/org/pentaho/di/trans/step/messages/messages_en_US.properties +++ b/engine/src/org/pentaho/di/trans/step/messages/messages_en_US.properties @@ -150,6 +150,7 @@ BaseStepDialog.ErrorHandling.NrErrField.Label=Nr of errors fieldname BaseStep.TypeLongDesc.LDAPInput=LDAP Input BaseStep.Log.OutputRowInfo=output rel. is {0}\:{1} BaseStep.TypeLongDesc.TextFileInput=Text file input +BaseStep.TypeLongDesc.OldTextFileInput=Old text file input BaseStep.Log.MaxPercentageRejectedReached=The maximum percentage of rejected rows of {0} has been reached. {1} rows where rejected out of {2}. This transformation is being asked to stop. BaseStep.TypeTooltipDesc.MergeRows=Merge two streams of rows, sorted on a certain key. The two streams are compared and the equals, changed, deleted and new rows are flagged. BaseStep.TypeTooltipDesc.StringCut=Strings cut (substring). diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStep.java b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStep.java new file mode 100644 index 000000000000..7000ca9c77b9 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStep.java @@ -0,0 +1,437 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.Result; +import org.pentaho.di.core.ResultFile; +import org.pentaho.di.core.RowSet; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.row.RowDataUtil; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStep; +import org.pentaho.di.trans.step.StepDataInterface; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.step.errorhandling.CompositeFileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerContentLineNumber; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerMissingFiles; + +/** + * This class contains base functionality for file-based input steps. + * + * @author Alexander Buloichik + */ +public abstract class BaseInputStep extends BaseStep + implements IBaseInputStepControl { + private static Class PKG = BaseInputStep.class; // for i18n purposes, needed by Translator2!! TODO: is + // it right for + // base class ??? + + protected M meta; + + protected D data; + + /** + * Content-dependent initialization. + */ + protected abstract boolean init(); + + /** + * Create reader for specific file. + */ + protected abstract IBaseInputReader createReader(M meta, D data, FileObject file) throws Exception; + + public BaseInputStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, + TransMeta transMeta, Trans trans) { + super(stepMeta, stepDataInterface, copyNr, transMeta, trans); + } + + /** + * Initialize step before execute. + */ + @Override + public boolean init(StepMetaInterface smi, StepDataInterface sdi) { + meta = (M) smi; + data = (D) sdi; + + if (!super.init(smi, sdi)) { + return false; + } + + initErrorHandling(); + + meta.additionalOutputFields.normalize(); + data.files = meta.getTextFileList(this); + data.currentFileIndex = 0; + + // If there are missing files, + // fail if we don't ignore errors + // + Result previousResult = getTrans().getPreviousResult(); + Map resultFiles = (previousResult != null) ? previousResult.getResultFiles() + : null; + + if ((previousResult == null || resultFiles == null || resultFiles.size() == 0) + && data.files.nrOfMissingFiles() > 0 && !meta.inputFiles.acceptingFilenames + && !meta.errorHandling.errorIgnored) { + logError(BaseMessages.getString(PKG, "TextFileInput.Log.Error.NoFilesSpecified")); + return false; + } + + String clusterSize = getVariable(Const.INTERNAL_VARIABLE_CLUSTER_SIZE); + if (!Const.isEmpty(clusterSize) && Integer.valueOf(clusterSize) > 1) { + // TODO: add metadata to configure this. + String nr = getVariable(Const.INTERNAL_VARIABLE_SLAVE_SERVER_NUMBER); + if (log.isDetailed()) { + logDetailed("Running on slave server #" + nr + + " : assuming that each slave reads a dedicated part of the same file(s)."); + } + } + + return init(); + } + + /** + * Open next VFS file for processing. + * + * This method will support different parallelization methods later. + */ + protected boolean openNextFile() { + try { + if (data.currentFileIndex >= data.files.nrOfFiles()) { + // all files already processed + return false; + } + + // Is this the last file? + data.file = data.files.getFile(data.currentFileIndex); + data.filename = KettleVFS.getFilename(data.file); + + fillFileAdditionalFields(data, data.file); + if (meta.inputFiles.passingThruFields) { + data.currentPassThruFieldsRow = data.passThruFields.get(data.file); + } + + // Add this files to the result of this transformation. + // + if (meta.inputFiles.isaddresult) { + ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.file, + getTransMeta().getName(), toString()); + resultFile.setComment("File was read by an Text File input step"); + addResultFile(resultFile); + } + if (log.isBasic()) { + logBasic("Opening file: " + data.file.getName().getFriendlyURI()); + } + + data.dataErrorLineHandler.handleFile(data.file); + + data.reader = createReader(meta, data, data.file); + } catch (Exception e) { + String errorMsg = "Couldn't open file #" + data.currentFileIndex + " : " + + data.file.getName().getFriendlyURI() + " --> " + e.toString(); + logError(errorMsg); + if (failAfterBadFile(errorMsg)) { // !meta.isSkipBadFiles()) stopAll(); + stopAll(); + } + setErrors(getErrors() + 1); + return false; + } + + // Move file pointer ahead! + data.currentFileIndex++; + + return true; + } + + /** + * Process next row. This methods opens next file automatically. + */ + @Override + public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { + meta = (M) smi; + data = (D) sdi; + + if (first) { + first = false; + prepareToRowProcessing(); + + if (!openNextFile()) { + setOutputDone(); // signal end to receiver(s) + closeLastFile(); + return false; + } + } + + while (true) { + if (data.reader.readRow()) { + // row processed + return true; + } + // end of current file + closeLastFile(); + + if (!openNextFile()) { + // there are no more files + break; + } + } + + // after all files processed + setOutputDone(); // signal end to receiver(s) + closeLastFile(); + return false; + } + + /** + * Prepare to process. Executed only first time row processing. It can't be possible to prepare to process + * in the init() phrase, because files can be in fields from previous step. + */ + protected void prepareToRowProcessing() throws KettleException { + data.outputRowMeta = new RowMeta(); + RowMetaInterface[] infoStep = null; + + if (meta.inputFiles.acceptingFilenames) { + // input files from previous step + infoStep = filesFromPreviousStep(); + } + + // get the metadata populated. Simple and easy. + meta.getFields(data.outputRowMeta, getStepname(), infoStep, null, this, repository, metaStore); + // Create convert meta-data objects that will contain Date & Number formatters + // + data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING); + + BaseStepUtils.handleMissingFiles(data.files, log, meta.errorHandling.errorIgnored, + data.dataErrorLineHandler); + + // Count the number of repeat fields... + for (int i = 0; i < meta.inputFiles.inputFields.length; i++) { + if (meta.inputFiles.inputFields[i].isRepeated()) { + data.nr_repeats++; + } + } + } + + @Override + public boolean checkFeedback(long lines) { + return super.checkFeedback(lines); + } + + /** + * Initialize error handling. + * + * TODO: should we set charset for error files from content meta ? What about case for automatic charset ? + */ + private void initErrorHandling() { + List dataErrorLineHandlers = new ArrayList(2); + if (meta.errorHandling.lineNumberFilesDestinationDirectory != null) { + dataErrorLineHandlers.add(new FileErrorHandlerContentLineNumber(getTrans().getCurrentDate(), + environmentSubstitute(meta.errorHandling.lineNumberFilesDestinationDirectory), + meta.errorHandling.lineNumberFilesExtension, meta.getEncoding(), this)); + } + if (meta.errorHandling.errorFilesDestinationDirectory != null) { + dataErrorLineHandlers.add(new FileErrorHandlerMissingFiles(getTrans().getCurrentDate(), + environmentSubstitute(meta.errorHandling.errorFilesDestinationDirectory), + meta.errorHandling.errorFilesExtension, meta.getEncoding(), this)); + } + data.dataErrorLineHandler = new CompositeFileErrorHandler(dataErrorLineHandlers); + } + + /** + * Read files from previous step. + */ + private RowMetaInterface[] filesFromPreviousStep() throws KettleException { + RowMetaInterface[] infoStep = null; + + data.files.getFiles().clear(); + + int idx = -1; + RowSet rowSet = findInputRowSet(meta.inputFiles.acceptingStepName); + + Object[] fileRow = getRowFrom(rowSet); + while (fileRow != null) { + RowMetaInterface prevInfoFields = rowSet.getRowMeta(); + if (idx < 0) { + if (meta.inputFiles.passingThruFields) { + data.passThruFields = new HashMap(); + infoStep = new RowMetaInterface[] { prevInfoFields }; + data.nrPassThruFields = prevInfoFields.size(); + } + idx = prevInfoFields.indexOfValue(meta.inputFiles.acceptingField); + if (idx < 0) { + logError(BaseMessages.getString(PKG, "TextFileInput.Log.Error.UnableToFindFilenameField", + meta.inputFiles.acceptingField)); + setErrors(getErrors() + 1); + stopAll(); + return null; + } + } + String fileValue = prevInfoFields.getString(fileRow, idx); + try { + FileObject fileObject = KettleVFS.getFileObject(fileValue, getTransMeta()); + data.files.addFile(fileObject); + if (meta.inputFiles.passingThruFields) { + data.passThruFields.put(fileObject, fileRow); + } + } catch (KettleFileException e) { + logError(BaseMessages.getString(PKG, "TextFileInput.Log.Error.UnableToCreateFileObject", + fileValue), e); + } + + // Grab another row + fileRow = getRowFrom(rowSet); + } + + if (data.files.nrOfFiles() == 0) { + if (log.isDetailed()) { + logDetailed(BaseMessages.getString(PKG, "TextFileInput.Log.Error.NoFilesSpecified")); + } + return null; + } + return infoStep; + } + + /** + * Close last opened file/ + */ + protected void closeLastFile() { + if (data.reader != null) { + try { + data.reader.close(); + } catch (Exception ex) { + failAfterBadFile("Error close reader"); + } + data.reader = null; + } + if (data.file != null) { + try { + data.file.close(); + } catch (Exception ex) { + failAfterBadFile("Error close file"); + } + data.file = null; + } + } + + /** + * Dispose step. + */ + @Override + public void dispose(StepMetaInterface smi, StepDataInterface sdi) { + closeLastFile(); + + super.dispose(smi, sdi); + } + + /** + * + * @param errorMsg + * Message to send to rejected row if enabled + * @return If should stop processing after having problems with a file + */ + public boolean failAfterBadFile(String errorMsg) { + + if (getStepMeta().isDoingErrorHandling() && data.filename != null + && !data.rejectedFiles.containsKey(data.filename)) { + data.rejectedFiles.put(data.filename, true); + rejectCurrentFile(errorMsg); + } + + return !meta.errorHandling.errorIgnored || !meta.errorHandling.skipBadFiles; + } + + /** + * Send file name and/or error message to error output + * + * @param errorMsg + * Message to send to rejected row if enabled + */ + private void rejectCurrentFile(String errorMsg) { + if (StringUtils.isNotBlank(meta.errorHandling.fileErrorField) + || StringUtils.isNotBlank(meta.errorHandling.fileErrorMessageField)) { + RowMetaInterface rowMeta = getInputRowMeta(); + if (rowMeta == null) { + rowMeta = new RowMeta(); + } + + int errorFileIndex = (StringUtils.isBlank(meta.errorHandling.fileErrorField)) ? -1 + : BaseStepUtils.addValueMeta(getStepname(), rowMeta, + this.environmentSubstitute(meta.errorHandling.fileErrorField)); + + int errorMessageIndex = StringUtils.isBlank(meta.errorHandling.fileErrorMessageField) ? -1 + : BaseStepUtils.addValueMeta(getStepname(), rowMeta, + this.environmentSubstitute(meta.errorHandling.fileErrorMessageField)); + + try { + Object[] rowData = getRow(); + if (rowData == null) { + rowData = RowDataUtil.allocateRowData(rowMeta.size()); + } + + if (errorFileIndex >= 0) { + rowData[errorFileIndex] = data.filename; + } + if (errorMessageIndex >= 0) { + rowData[errorMessageIndex] = errorMsg; + } + + putError(rowMeta, rowData, getErrors(), data.filename, null, "ERROR_CODE"); + } catch (Exception e) { + logError("Error sending error row", e); + } + } + } + + /** + * Prepare file-dependent data for fill additional fields. + */ + protected void fillFileAdditionalFields(D data, FileObject file) throws FileSystemException { + data.shortFilename = file.getName().getBaseName(); + data.path = KettleVFS.getFilename(file.getParent()); + data.hidden = file.isHidden(); + data.extension = file.getName().getExtension(); + data.lastModificationDateTime = new Date(file.getContent().getLastModifiedTime()); + data.uriName = file.getName().getURI(); + data.rootUriName = file.getName().getRootURI(); + data.size = file.getContent().getSize(); + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepData.java b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepData.java new file mode 100644 index 000000000000..f34daa60215b --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepData.java @@ -0,0 +1,80 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.trans.step.BaseStepData; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; + +/** + * Some base data for file-based input steps. + * + * @author Alexander Buloichik + */ +public class BaseInputStepData extends BaseStepData { + + public FileErrorHandler dataErrorLineHandler; + + /** Files for process. */ + public FileInputList files; + + /** Current file info. */ + public String filename; + public int currentFileIndex; + public FileObject file; + + /** Reader for current file. */ + public IBaseInputReader reader; + + public RowMetaInterface outputRowMeta; + + public HashMap passThruFields; + + public Object[] currentPassThruFieldsRow; + + public int nrPassThruFields; + + public RowMetaInterface convertRowMeta; + + public int nr_repeats; + // public boolean isLastFile; + + public Map rejectedFiles = new HashMap(); + + /** File-dependent data for fill additional fields. */ + public String shortFilename; + public String path; + public String extension; + public boolean hidden; + public Date lastModificationDateTime; + public String uriName; + public String rootUriName; + public long size; + +} diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepMeta.java b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepMeta.java new file mode 100644 index 000000000000..2c985cccfcfd --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseInputStepMeta.java @@ -0,0 +1,282 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.resource.ResourceEntry; +import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.resource.ResourceEntry.ResourceType; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStepMeta; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; + +/** + * Base meta for file-based input steps. + * + * @author Alexander Buloichik + */ +public abstract class BaseInputStepMeta extends BaseStepMeta { + private Class PKG = this.getClass(); // for i18n purposes, needed by Translator2!! + + public static final String[] RequiredFilesCode = new String[] { "N", "Y" }; + + public static final String NO = "N"; + + public static final String YES = "Y"; + + public final String[] RequiredFilesDesc = new String[] { BaseMessages.getString(PKG, "System.Combo.No"), + BaseMessages.getString(PKG, "System.Combo.Yes") }; + + public InputFiles inputFiles = new InputFiles(); + public ErrorHandling errorHandling = new ErrorHandling(); + public AdditionalOutputFields additionalOutputFields = new AdditionalOutputFields(); + + /** + * Input files settings. + */ + public static class InputFiles implements Cloneable { + + /** Array of filenames */ + public String[] fileName; + + /** Wildcard or filemask (regular expression) */ + public String[] fileMask; + + /** Wildcard or filemask to exclude (regular expression) */ + public String[] excludeFileMask; + + /** Array of boolean values as string, indicating if a file is required. */ + public String[] fileRequired; + + /** Array of boolean values as string, indicating if we need to fetch sub folders. */ + public String[] includeSubFolders; + + /** Are we accepting filenames in input rows? */ + public boolean acceptingFilenames; + + /** The stepname to accept filenames from */ + public String acceptingStepName; + + /** If receiving input rows, should we pass through existing fields? */ + public boolean passingThruFields; + + /** The field in which the filename is placed */ + public String acceptingField; + + /** The fields to import... */ + public TextFileInputField[] inputFields; + + /** The add filenames to result filenames flag */ + public boolean isaddresult; + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + } + + /** + * Error handling settings. + */ + public static class ErrorHandling implements Cloneable { + + /** Ignore error : turn into warnings */ + public boolean errorIgnored; + + /** File error field name. */ + public String fileErrorField; + + /** File error text field name. */ + public String fileErrorMessageField; + + public boolean skipBadFiles; + + /** The directory that will contain warning files */ + public String warningFilesDestinationDirectory; + + /** The extension of warning files */ + public String warningFilesExtension; + + /** The directory that will contain error files */ + public String errorFilesDestinationDirectory; + + /** The extension of error files */ + public String errorFilesExtension; + + /** The directory that will contain line number files */ + public String lineNumberFilesDestinationDirectory; + + /** The extension of line number files */ + public String lineNumberFilesExtension; + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + } + + /** + * Additional fields settings. + */ + public static class AdditionalOutputFields implements Cloneable { + + /** Additional fields **/ + public String shortFilenameField; + public String extensionField; + public String pathField; + public String sizeField; + public String hiddenField; + public String lastModificationField; + public String uriField; + public String rootUriField; + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + + /** + * Set null for all empty field values to be able to fast check during step processing. Need to be + * executed once before processing. + */ + public void normalize() { + if (StringUtils.isBlank(shortFilenameField)) { + shortFilenameField = null; + } + if (StringUtils.isBlank(extensionField)) { + extensionField = null; + } + if (StringUtils.isBlank(pathField)) { + pathField = null; + } + if (StringUtils.isBlank(sizeField)) { + sizeField = null; + } + if (StringUtils.isBlank(hiddenField)) { + hiddenField = null; + } + if (StringUtils.isBlank(lastModificationField)) { + lastModificationField = null; + } + if (StringUtils.isBlank(uriField)) { + uriField = null; + } + if (StringUtils.isBlank(rootUriField)) { + rootUriField = null; + } + } + } + + public Object clone() { + BaseInputStepMeta retval = (BaseInputStepMeta) super.clone(); + + retval.inputFiles = (InputFiles) inputFiles.clone(); + retval.errorHandling = (ErrorHandling) errorHandling.clone(); + retval.additionalOutputFields = (AdditionalOutputFields) additionalOutputFields.clone(); + + return retval; + } + + /** + * @param fileRequired + * The fileRequired to set. + */ + public void inputFiles_fileRequired(String[] fileRequiredin) { + for (int i = 0; i < fileRequiredin.length; i++) { + inputFiles.fileRequired[i] = getRequiredFilesCode(fileRequiredin[i]); + } + } + + public String[] inputFiles_includeSubFolders() { + return inputFiles.includeSubFolders; + } + + public void inputFiles_includeSubFolders(String[] includeSubFoldersin) { + for (int i = 0; i < includeSubFoldersin.length; i++) { + inputFiles.includeSubFolders[i] = getRequiredFilesCode(includeSubFoldersin[i]); + } + } + + public String getRequiredFilesCode(String tt) { + if (tt == null) { + return RequiredFilesCode[0]; + } + if (tt.equals(RequiredFilesDesc[1])) { + return RequiredFilesCode[1]; + } else { + return RequiredFilesCode[0]; + } + } + + public FileInputList getTextFileList(VariableSpace space) { + return FileInputList.createFileList(space, inputFiles.fileName, inputFiles.fileMask, + inputFiles.excludeFileMask, inputFiles.fileRequired, includeSubFolderBoolean()); + } + + private boolean[] includeSubFolderBoolean() { + int len = inputFiles.fileName.length; + boolean[] includeSubFolderBoolean = new boolean[len]; + for (int i = 0; i < len; i++) { + includeSubFolderBoolean[i] = YES.equalsIgnoreCase(inputFiles.includeSubFolders[i]); + } + return includeSubFolderBoolean; + } + + @Override + public List getResourceDependencies(TransMeta transMeta, StepMeta stepInfo) { + List references = new ArrayList(5); + ResourceReference reference = new ResourceReference(stepInfo); + references.add(reference); + + String[] textFiles = getFilePaths(transMeta); + if (textFiles != null) { + for (int i = 0; i < textFiles.length; i++) { + reference.getEntries().add(new ResourceEntry(textFiles[i], ResourceType.FILE)); + } + } + return references; + } + + public String[] getFilePaths(VariableSpace space) { + return FileInputList.createFilePathList(space, inputFiles.fileName, inputFiles.fileMask, + inputFiles.excludeFileMask, inputFiles.fileRequired, includeSubFolderBoolean()); + } + + abstract public String getEncoding(); +} diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/BaseStepUtils.java b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseStepUtils.java new file mode 100644 index 000000000000..33c900203806 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/BaseStepUtils.java @@ -0,0 +1,97 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import java.util.List; + +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; + +/** + * Utils for file-based input steps. + * + * @author Alexander Buloichik + */ +public class BaseStepUtils { + + public static void handleMissingFiles( FileInputList files, LogChannelInterface log, boolean isErrorIgnored, + FileErrorHandler errorHandler ) throws KettleException { + List nonExistantFiles = files.getNonExistantFiles(); + + if ( !nonExistantFiles.isEmpty() ) { + String message = FileInputList.getRequiredFilesDescription( nonExistantFiles ); + if ( log.isBasic() ) { + log.logBasic( "Required files", "WARNING: Missing " + message ); + } + if ( isErrorIgnored ) { + for ( FileObject fileObject : nonExistantFiles ) { + errorHandler.handleNonExistantFile( fileObject ); + } + } else { + throw new KettleException( "Following required files are missing: " + message ); + } + } + + List nonAccessibleFiles = files.getNonAccessibleFiles(); + if ( !nonAccessibleFiles.isEmpty() ) { + String message = FileInputList.getRequiredFilesDescription( nonAccessibleFiles ); + if ( log.isBasic() ) { + log.logBasic( "Required files", "WARNING: Not accessible " + message ); + } + if ( isErrorIgnored ) { + for ( FileObject fileObject : nonAccessibleFiles ) { + errorHandler.handleNonAccessibleFile( fileObject ); + } + } else { + throw new KettleException( "Following required files are not accessible: " + message ); + } + } + } + + /** + * Adds String value meta with given name if not present and returns index + * + * @param rowMeta + * @param fieldName + * @return Index in row meta of value meta with fieldName + */ + public static int addValueMeta( String stepName, RowMetaInterface rowMeta, String fieldName ) { + ValueMetaInterface valueMeta = new ValueMeta( fieldName, ValueMetaInterface.TYPE_STRING ); + valueMeta.setOrigin( stepName ); + // add if doesn't exist + int index = -1; + if ( !rowMeta.exists( valueMeta ) ) { + index = rowMeta.size(); + rowMeta.addValueMeta( valueMeta ); + } else { + index = rowMeta.indexOfValue( fieldName ); + } + return index; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputReader.java b/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputReader.java new file mode 100644 index 000000000000..1e9b1022cedb --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputReader.java @@ -0,0 +1,34 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import java.io.Closeable; + +import org.pentaho.di.core.exception.KettleException; + +/** + * Content-based reader for file. + */ +public interface IBaseInputReader extends Closeable { + boolean readRow() throws KettleException; +} diff --git a/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputStepControl.java b/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputStepControl.java new file mode 100644 index 000000000000..ce798c569090 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/baseinput/IBaseInputStepControl.java @@ -0,0 +1,48 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.baseinput; + +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.row.RowMetaInterface; + +/** + * Interface for some step operations required for parse input file. + */ +public interface IBaseInputStepControl { + long incrementLinesInput(); + + long getLinesWritten(); + + void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException; + + long getLinesInput(); + + boolean checkFeedback( long lines ); + + long incrementLinesUpdated(); + + boolean failAfterBadFile( String errorMsg ); + void stopAll() ; + long getErrors(); + void setErrors( long e ); +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldEncodingType.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldEncodingType.java new file mode 100644 index 000000000000..9789b8c4257d --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldEncodingType.java @@ -0,0 +1,124 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.io.UnsupportedEncodingException; + +import org.pentaho.di.core.Const; + +public enum OldEncodingType { + SINGLE( 1, 0, '\r', '\n' ), DOUBLE_BIG_ENDIAN( 2, 0xFEFF, 0x000d, 0x000a ), DOUBLE_LITTLE_ENDIAN( + 2, 0xFFFE, 0x0d00, 0x0a00 ); + + private int length; + + /** + * Byte Order Mark (BOM): http://en.wikipedia.org/wiki/Byte_Order_Mark + */ + private int bom; + private int carriageReturnChar; + private int lineFeedChar; + + /** + * @param length + * @param bom + */ + private OldEncodingType( int length, int bom, int carriageReturnChar, int lineFeedChar ) { + this.length = length; + this.bom = bom; + this.carriageReturnChar = carriageReturnChar; + this.lineFeedChar = lineFeedChar; + } + + public int getLength() { + return length; + } + + public int getBom() { + return bom; + } + + public int getCarriageReturnChar() { + return carriageReturnChar; + } + + public int getLineFeedChar() { + return lineFeedChar; + } + + public boolean isReturn( int c ) { + return c == carriageReturnChar || c == '\r'; + } + + public boolean isLinefeed( int c ) { + return c == lineFeedChar || c == '\n'; + } + + public static OldEncodingType guessEncodingType( String encoding ) { + + OldEncodingType encodingType; + + if ( Const.isEmpty( encoding ) ) { + encodingType = OldEncodingType.SINGLE; + } else if ( encoding.startsWith( "UnicodeBig" ) || encoding.equals( "UTF-16BE" ) ) { + encodingType = OldEncodingType.DOUBLE_BIG_ENDIAN; + } else if ( encoding.startsWith( "UnicodeLittle" ) || encoding.equals( "UTF-16LE" ) ) { + encodingType = OldEncodingType.DOUBLE_LITTLE_ENDIAN; + } else if ( encoding.equals( "UTF-16" ) ) { + encodingType = OldEncodingType.DOUBLE_BIG_ENDIAN; // The default, no BOM + } else { + encodingType = OldEncodingType.SINGLE; + } + + return encodingType; + } + + public byte[] getBytes( String string, String encoding ) throws UnsupportedEncodingException { + byte[] withBom; + if ( Const.isEmpty( encoding ) ) { + withBom = string.getBytes(); + } else { + withBom = string.getBytes( encoding ); + } + + switch ( length ) { + case 1: + return withBom; + case 2: + if ( withBom.length < 2 ) { + return withBom; + } + if ( withBom[0] < 0 && withBom[1] < 0 ) { + byte[] b = new byte[withBom.length - 2]; + for ( int i = 0; i < withBom.length - 2; i++ ) { + b[i] = withBom[i + 2]; + } + return b; + } else { + return withBom; + } + default: + return withBom; + } + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilter.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilter.java new file mode 100644 index 000000000000..14287756cc4a --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilter.java @@ -0,0 +1,128 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +public class OldTextFileFilter implements Cloneable { + /** The position of the occurrence of the filter string to check at */ + private int filterPosition; + + /** The string to filter on */ + private String filterString; + + /** True if we want to stop when we reach a filter line */ + private boolean filterLastLine; + + /** True if we want to match only this lines */ + private boolean filterPositive; + + /** + * @param filterPosition + * The position of the occurrence of the filter string to check at + * @param filterString + * The string to filter on + * @param filterLastLine + * True if we want to stop when we reach a filter string on the specified position False if you just want to + * skip the line. + * @param filterPositive + * True if we want to get only lines that match this string + * + */ + public OldTextFileFilter( int filterPosition, String filterString, boolean filterLastLine, boolean filterPositive ) { + this.filterPosition = filterPosition; + this.filterString = filterString; + this.filterLastLine = filterLastLine; + this.filterPositive = filterPositive; + } + + public OldTextFileFilter() { + } + + public Object clone() { + try { + Object retval = super.clone(); + return retval; + } catch ( CloneNotSupportedException e ) { + return null; + } + } + + /** + * @return Returns the filterLastLine. + */ + public boolean isFilterLastLine() { + return filterLastLine; + } + + /** + * @param filterLastLine + * The filterLastLine to set. + */ + public void setFilterLastLine( boolean filterLastLine ) { + this.filterLastLine = filterLastLine; + } + + /** + * @return Returns the filterPositive. + */ + public boolean isFilterPositive() { + return filterPositive; + } + + /** + * @param filterPositive + * The filterPositive to set. + */ + public void setFilterPositive( boolean filterPositive ) { + this.filterPositive = filterPositive; + } + + /** + * @return Returns the filterPosition. + */ + public int getFilterPosition() { + return filterPosition; + } + + /** + * @param filterPosition + * The filterPosition to set. + */ + public void setFilterPosition( int filterPosition ) { + this.filterPosition = filterPosition; + } + + /** + * @return Returns the filterString. + */ + public String getFilterString() { + return filterString; + } + + /** + * @param filterString + * The filterString to set. + */ + public void setFilterString( String filterString ) { + this.filterString = filterString; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilterProcessor.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilterProcessor.java new file mode 100644 index 000000000000..ea110bf5bde5 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileFilterProcessor.java @@ -0,0 +1,119 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +/** + * Processor of Filters. Kind of inversion principle, and to make unit testing easier. + * + * @author Sven Boden + */ +public class OldTextFileFilterProcessor { + + /** The filters to process */ + private OldTextFileFilter[] filters; + private boolean stopProcessing; + + /** + * @param filters + * The filters to process + */ + public OldTextFileFilterProcessor( OldTextFileFilter[] filters ) { + this.filters = filters; + this.stopProcessing = false; + + if ( filters.length == 0 ) { + // This makes processing faster in case there are no filters. + filters = null; + } + } + + public boolean doFilters( String line ) { + if ( filters == null ) { + return true; + } + + boolean filterOK = true; // if false: skip this row + boolean positiveMode = false; + boolean positiveMatchFound = false; + + // If we have at least one positive filter, we enter positiveMode + // Negative filters will always take precendence, meaning that the line + // is skipped if one of them is found + + for ( int f = 0; f < filters.length && filterOK; f++ ) { + OldTextFileFilter filter = filters[f]; + if ( filter.isFilterPositive() ) { + positiveMode = true; + } + + if ( filter.getFilterString() != null && filter.getFilterString().length() > 0 ) { + int from = filter.getFilterPosition(); + if ( from >= 0 ) { + int to = from + filter.getFilterString().length(); + if ( line.length() >= from && line.length() >= to ) { + String sub = line.substring( filter.getFilterPosition(), to ); + if ( sub.equalsIgnoreCase( filter.getFilterString() ) ) { + if ( filter.isFilterPositive() ) { + positiveMatchFound = true; + } else { + filterOK = false; // skip this one! + } + } + } + } else { // anywhere on the line + int idx = line.indexOf( filter.getFilterString() ); + if ( idx >= 0 ) { + if ( filter.isFilterPositive() ) { + positiveMatchFound = true; + } else { + filterOK = false; // skip this one! + } + } + } + + if ( !filterOK ) { + boolean isFilterLastLine = filter.isFilterLastLine(); + if ( isFilterLastLine ) { + stopProcessing = true; + } + } + } + } + + // Positive mode and no match found? Discard the line + if ( filterOK && positiveMode && !positiveMatchFound ) { + filterOK = false; + } + + return filterOK; + } + + /** + * Was processing requested to be stopped. Can only be true when doFilters was false. + * + * @return == true: processing should stop, == false: processing should continue. + */ + public boolean isStopProcessing() { + return stopProcessing; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInput.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInput.java new file mode 100644 index 000000000000..838667750fd5 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInput.java @@ -0,0 +1,1631 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.io.BufferedInputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.Result; +import org.pentaho.di.core.ResultFile; +import org.pentaho.di.core.compress.CompressionProvider; +import org.pentaho.di.core.compress.CompressionProviderFactory; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.playlist.FilePlayListAll; +import org.pentaho.di.core.playlist.FilePlayListReplay; +import org.pentaho.di.core.row.RowDataUtil; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.util.StringUtil; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.variables.Variables; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStep; +import org.pentaho.di.trans.step.StepDataInterface; +import org.pentaho.di.trans.step.StepInterface; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.step.errorhandling.AbstractFileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.CompositeFileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerContentLineNumber; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerMissingFiles; +import org.pentaho.di.trans.steps.textfileinput.InputFileMetaInterface; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; + + +/** + * Read all sorts of text files, convert them to rows and writes these to one or more output streams. + * + * @author Matt + * @since 4-apr-2003 + */ +public class OldTextFileInput extends BaseStep implements StepInterface { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + private static final int BUFFER_SIZE_INPUT_STREAM = 500; + + private OldTextFileInputMeta meta; + + private OldTextFileInputData data; + + private long lineNumberInFile; + + public OldTextFileInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, + Trans trans ) { + super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); + } + + public static final String getLine( LogChannelInterface log, InputStreamReader reader, int formatNr, + StringBuilder line ) throws KettleFileException { + OldEncodingType type = OldEncodingType.guessEncodingType( reader.getEncoding() ); + return getLine( log, reader, type, formatNr, line ); + } + + public static final String getLine( LogChannelInterface log, InputStreamReader reader, OldEncodingType encodingType, + int formatNr, StringBuilder line ) throws KettleFileException { + int c = 0; + line.setLength( 0 ); + try { + switch ( formatNr ) { + case OldTextFileInputMeta.FILE_FORMAT_DOS: + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isReturn( c ) || encodingType.isLinefeed( c ) ) { + c = reader.read(); // skip \n and \r + if ( !encodingType.isReturn( c ) && !encodingType.isLinefeed( c ) ) { + // make sure its really a linefeed or cariage return + // raise an error this is not a DOS file + // so we have pulled a character from the next line + throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.SingleLineFound" ) ); + } + return line.toString(); + } + if ( c >= 0 ) { + line.append( (char) c ); + } + } + break; + case OldTextFileInputMeta.FILE_FORMAT_UNIX: + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isLinefeed( c ) || encodingType.isReturn( c ) ) { + return line.toString(); + } + if ( c >= 0 ) { + line.append( (char) c ); + } + } + break; + case OldTextFileInputMeta.FILE_FORMAT_MIXED: + // in mixed mode we suppose the LF is the last char and CR is ignored + // not for MAC OS 9 but works for Mac OS X. Mac OS 9 can use UNIX-Format + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isLinefeed( c ) ) { + return line.toString(); + } else if ( !encodingType.isReturn( c ) ) { + if ( c >= 0 ) { + line.append( (char) c ); + } + } + } + break; + default: + break; + } + } catch ( KettleFileException e ) { + throw e; + } catch ( Exception e ) { + if ( line.length() == 0 ) { + throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ExceptionReadingLine", e + .toString() ), e ); + } + return line.toString(); + } + if ( line.length() > 0 ) { + return line.toString(); + } + + return null; + } + + @Deprecated + public static final String[] guessStringsFromLine( LogChannelInterface log, String line, OldTextFileInputMeta inf, + String delimiter ) throws KettleException { + return guessStringsFromLine( new Variables(), log, line, inf, delimiter, StringUtil.substituteHex( inf + .getEnclosure() ), StringUtil.substituteHex( inf.getEscapeCharacter() ) ); + } + + public static final String[] guessStringsFromLine( VariableSpace space, LogChannelInterface log, String line, + OldTextFileInputMeta inf, String delimiter, String enclosure, String escapeCharacter ) throws KettleException { + List strings = new ArrayList(); + + String pol; // piece of line + + try { + if ( line == null ) { + return null; + } + + if ( inf.getFileType().equalsIgnoreCase( "CSV" ) ) { + + // Split string in pieces, only for CSV! + + int pos = 0; + int length = line.length(); + boolean dencl = false; + + int len_encl = ( enclosure == null ? 0 : enclosure.length() ); + int len_esc = ( escapeCharacter == null ? 0 : escapeCharacter.length() ); + + while ( pos < length ) { + int from = pos; + int next; + + boolean encl_found; + boolean contains_escaped_enclosures = false; + boolean contains_escaped_separators = false; + + // Is the field beginning with an enclosure? + // "aa;aa";123;"aaa-aaa";000;... + if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.ConvertLineToRow", line.substring( from, from + len_encl ) ) ); + } + encl_found = true; + int p = from + len_encl; + + boolean is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ) + .equalsIgnoreCase( enclosure ); + boolean is_escape = + len_esc > 0 && p + len_esc < length + && line.substring( p, p + len_esc ).equalsIgnoreCase( escapeCharacter ); + + boolean enclosure_after = false; + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equalsIgnoreCase( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; + } + } + } + + // Look for a closing enclosure! + while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { + p++; + enclosure_after = false; + is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equals( enclosure ); + is_escape = + len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equals( escapeCharacter ); + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equals( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; // remember + } + } + } + } + + if ( p >= length ) { + next = p; + } else { + next = p + len_encl; + } + + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); + } + } else { + encl_found = false; + boolean found = false; + int startpoint = from; + // int tries = 1; + do { + next = line.indexOf( delimiter, startpoint ); + + // See if this position is preceded by an escape character. + if ( len_esc > 0 && next - len_esc > 0 ) { + String before = line.substring( next - len_esc, next ); + + if ( escapeCharacter.equals( before ) ) { + // take the next separator, this one is escaped... + startpoint = next + 1; + // tries++; + contains_escaped_separators = true; + } else { + found = true; + } + } else { + found = true; + } + } while ( !found && next >= 0 ); + } + if ( next == -1 ) { + next = length; + } + + if ( encl_found ) { + pol = line.substring( from + len_encl, next - len_encl ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); + } + } else { + pol = line.substring( from, next ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); + } + } + + if ( dencl ) { + StringBuilder sbpol = new StringBuilder( pol ); + int idx = sbpol.indexOf( enclosure + enclosure ); + while ( idx >= 0 ) { + sbpol.delete( idx, idx + enclosure.length() ); + idx = sbpol.indexOf( enclosure + enclosure ); + } + pol = sbpol.toString(); + } + + // replace the escaped enclosures with enclosures... + if ( contains_escaped_enclosures ) { + String replace = escapeCharacter + enclosure; + String replaceWith = enclosure; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // replace the escaped separators with separators... + if ( contains_escaped_separators ) { + String replace = escapeCharacter + delimiter; + String replaceWith = delimiter; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // Now add pol to the strings found! + strings.add( pol ); + + pos = next + delimiter.length(); + } + if ( pos == length ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); + } + strings.add( "" ); + } + } else { + // Fixed file format: Simply get the strings at the required positions... + for ( int i = 0; i < inf.getInputFields().length; i++ ) { + TextFileInputField field = inf.getInputFields()[i]; + + int length = line.length(); + + if ( field.getPosition() + field.getLength() <= length ) { + strings.add( line.substring( field.getPosition(), field.getPosition() + field.getLength() ) ); + } else { + if ( field.getPosition() < length ) { + strings.add( line.substring( field.getPosition() ) ); + } else { + strings.add( "" ); + } + } + } + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e + .toString() ), e ); + } + + return strings.toArray( new String[strings.size()] ); + } + + public static final String[] convertLineToStrings( LogChannelInterface log, String line, InputFileMetaInterface inf, + String delimiter, String enclosure, String escapeCharacters ) throws KettleException { + String[] strings = new String[inf.getInputFields().length]; + int fieldnr; + + String pol; // piece of line + + try { + if ( line == null ) { + return null; + } + + if ( inf.getFileType().equalsIgnoreCase( "CSV" ) ) { + // Split string in pieces, only for CSV! + + fieldnr = 0; + int pos = 0; + int length = line.length(); + boolean dencl = false; + + int len_encl = ( enclosure == null ? 0 : enclosure.length() ); + int len_esc = ( escapeCharacters == null ? 0 : escapeCharacters.length() ); + + while ( pos < length ) { + int from = pos; + int next; + + boolean encl_found; + boolean contains_escaped_enclosures = false; + boolean contains_escaped_separators = false; + + // Is the field beginning with an enclosure? + // "aa;aa";123;"aaa-aaa";000;... + if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.Encloruse", line.substring( from, from + len_encl ) ) ); + } + encl_found = true; + int p = from + len_encl; + + boolean is_enclosure = + len_encl > 0 && p + len_encl < length + && line.substring( p, p + len_encl ).equalsIgnoreCase( enclosure ); + boolean is_escape = + len_esc > 0 && p + len_esc < length + && line.substring( p, p + len_esc ).equalsIgnoreCase( inf.getEscapeCharacter() ); + + boolean enclosure_after = false; + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equalsIgnoreCase( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; + } + } + } + + // Look for a closing enclosure! + while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { + p++; + enclosure_after = false; + is_enclosure = + len_encl > 0 && p + len_encl < length + && line.substring( p, p + len_encl ).equals( enclosure ); + is_escape = + len_esc > 0 && p + len_esc < length + && line.substring( p, p + len_esc ).equals( inf.getEscapeCharacter() ); + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equals( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; // remember + } + } + } + } + + if ( p >= length ) { + next = p; + } else { + next = p + len_encl; + } + + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); + } + } else { + encl_found = false; + boolean found = false; + int startpoint = from; + // int tries = 1; + do { + next = line.indexOf( delimiter, startpoint ); + + // See if this position is preceded by an escape character. + if ( len_esc > 0 && next - len_esc > 0 ) { + String before = line.substring( next - len_esc, next ); + + if ( inf.getEscapeCharacter().equals( before ) ) { + // take the next separator, this one is escaped... + startpoint = next + 1; + // tries++; + contains_escaped_separators = true; + } else { + found = true; + } + } else { + found = true; + } + } while ( !found && next >= 0 ); + } + if ( next == -1 ) { + next = length; + } + + if ( encl_found && ( ( from + len_encl ) <= ( next - len_encl ) ) ) { + pol = line.substring( from + len_encl, next - len_encl ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); + } + } else { + pol = line.substring( from, next ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); + } + } + + if ( dencl && Const.isEmpty( inf.getEscapeCharacter() ) ) { + StringBuilder sbpol = new StringBuilder( pol ); + int idx = sbpol.indexOf( enclosure + enclosure ); + while ( idx >= 0 ) { + sbpol.delete( idx, idx + enclosure.length() ); + idx = sbpol.indexOf( enclosure + enclosure ); + } + pol = sbpol.toString(); + } + + // replace the escaped enclosures with enclosures... + if ( contains_escaped_enclosures ) { + String replace = inf.getEscapeCharacter() + enclosure; + String replaceWith = enclosure; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // replace the escaped separators with separators... + if ( contains_escaped_separators ) { + String replace = inf.getEscapeCharacter() + delimiter; + String replaceWith = delimiter; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // Now add pol to the strings found! + try { + strings[fieldnr] = pol; + } catch ( ArrayIndexOutOfBoundsException e ) { + // In case we didn't allocate enough space. + // This happens when you have less header values specified than there are actual values in the rows. + // As this is "the exception" we catch and resize here. + // + String[] newStrings = new String[strings.length]; + for ( int x = 0; x < strings.length; x++ ) { + newStrings[x] = strings[x]; + } + strings = newStrings; + } + + pos = next + delimiter.length(); + fieldnr++; + } + if ( pos == length ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); + } + if ( fieldnr < strings.length ) { + strings[fieldnr] = Const.EMPTY_STRING; + } + fieldnr++; + } + } else { + // Fixed file format: Simply get the strings at the required positions... + for ( int i = 0; i < inf.getInputFields().length; i++ ) { + TextFileInputField field = inf.getInputFields()[i]; + + int length = line.length(); + + if ( field.getPosition() + field.getLength() <= length ) { + strings[i] = line.substring( field.getPosition(), field.getPosition() + field.getLength() ); + } else { + if ( field.getPosition() < length ) { + strings[i] = line.substring( field.getPosition() ); + } else { + strings[i] = ""; + } + } + } + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e + .toString() ), e ); + } + + return strings; + } + + /** + * @deprecated Use {@link #convertLineToRow(LogChannelInterface, OldTextFileLine, + * OldInputFileMetaInterface, Object[], int, RowMetaInterface,RowMetaInterface, + * String, long, String, String, String, FileErrorHandler, boolean, boolean, + * boolean, boolean, boolean, boolean, boolean, boolean, String, String, boolean, + * Date, String, String, String, long)} instead. + */ + @Deprecated + public static final Object[] convertLineToRow( LogChannelInterface log, OldTextFileLine textFileLine, + InputFileMetaInterface info, RowMetaInterface outputRowMeta, RowMetaInterface convertRowMeta, String fname, + long rowNr, String delimiter, FileErrorHandler errorHandler, boolean addShortFilename, boolean addExtension, + boolean addPath, boolean addSize, boolean addIsHidden, boolean addLastModificationDate, boolean addUri, + boolean addRootUri, String shortFilename, String path, boolean hidden, Date modificationDateTime, String uri, + String rooturi, String extension, long size ) throws KettleException { + return convertLineToRow( log, textFileLine, info, null, 0, outputRowMeta, convertRowMeta, fname, rowNr, delimiter, + StringUtil.substituteHex( info.getEnclosure() ), StringUtil.substituteHex( info.getEscapeCharacter() ), + errorHandler, addShortFilename, addExtension, addPath, addSize, addIsHidden, addLastModificationDate, addUri, + addRootUri, shortFilename, path, hidden, modificationDateTime, uri, rooturi, extension, size ); + } + + public static final Object[] convertLineToRow( LogChannelInterface log, OldTextFileLine textFileLine, + InputFileMetaInterface info, Object[] passThruFields, int nrPassThruFields, RowMetaInterface outputRowMeta, + RowMetaInterface convertRowMeta, String fname, long rowNr, String delimiter, String enclosure, + String escapeCharacter, FileErrorHandler errorHandler, boolean addShortFilename, boolean addExtension, + boolean addPath, boolean addSize, boolean addIsHidden, boolean addLastModificationDate, boolean addUri, + boolean addRootUri, String shortFilename, String path, boolean hidden, Date modificationDateTime, String uri, + String rooturi, String extension, long size ) throws KettleException { + if ( textFileLine == null || textFileLine.line == null ) { + return null; + } + + Object[] r = RowDataUtil.allocateRowData( outputRowMeta.size() ); // over-allocate a bit in the row producing + // steps... + + int nrfields = info.getInputFields().length; + int fieldnr; + + Long errorCount = null; + if ( info.isErrorIgnored() && info.getErrorCountField() != null && info.getErrorCountField().length() > 0 ) { + errorCount = new Long( 0L ); + } + String errorFields = null; + if ( info.isErrorIgnored() && info.getErrorFieldsField() != null && info.getErrorFieldsField().length() > 0 ) { + errorFields = ""; + } + String errorText = null; + if ( info.isErrorIgnored() && info.getErrorTextField() != null && info.getErrorTextField().length() > 0 ) { + errorText = ""; + } + + try { + // System.out.println("Convertings line to string ["+line+"]"); + String[] strings = convertLineToStrings( log, textFileLine.line, info, delimiter, enclosure, escapeCharacter ); + int shiftFields = ( passThruFields == null ? 0 : nrPassThruFields ); + for ( fieldnr = 0; fieldnr < nrfields; fieldnr++ ) { + TextFileInputField f = info.getInputFields()[fieldnr]; + int valuenr = shiftFields + fieldnr; + ValueMetaInterface valueMeta = outputRowMeta.getValueMeta( valuenr ); + ValueMetaInterface convertMeta = convertRowMeta.getValueMeta( valuenr ); + + Object value; + + String nullif = fieldnr < nrfields ? f.getNullString() : ""; + String ifnull = fieldnr < nrfields ? f.getIfNullValue() : ""; + int trim_type = fieldnr < nrfields ? f.getTrimType() : ValueMetaInterface.TRIM_TYPE_NONE; + + if ( fieldnr < strings.length ) { + String pol = strings[fieldnr]; + try { + value = valueMeta.convertDataFromString( pol, convertMeta, nullif, ifnull, trim_type ); + } catch ( Exception e ) { + // OK, give some feedback! + String message = + BaseMessages.getString( PKG, "TextFileInput.Log.CoundNotParseField", valueMeta.toStringMeta(), + "" + pol, valueMeta.getConversionMask(), "" + rowNr ); + + if ( info.isErrorIgnored() ) { + log.logDetailed( fname, BaseMessages.getString( PKG, "TextFileInput.Log.Warning" ) + ": " + message + + " : " + e.getMessage() ); + + value = null; + + if ( errorCount != null ) { + errorCount = new Long( errorCount.longValue() + 1L ); + } + if ( errorFields != null ) { + StringBuilder sb = new StringBuilder( errorFields ); + if ( sb.length() > 0 ) { + sb.append( "\t" ); // TODO document this change + } + sb.append( valueMeta.getName() ); + errorFields = sb.toString(); + } + if ( errorText != null ) { + StringBuilder sb = new StringBuilder( errorText ); + if ( sb.length() > 0 ) { + sb.append( Const.CR ); + } + sb.append( message ); + errorText = sb.toString(); + } + if ( errorHandler != null ) { + errorHandler.handleLineError( textFileLine.lineNumber, AbstractFileErrorHandler.NO_PARTS ); + } + + if ( info.isErrorLineSkipped() ) { + r = null; // compensates for stmt: r.setIgnore(); + } + } else { + throw new KettleException( message, e ); + } + } + } else { + // No data found: TRAILING NULLCOLS: add null value... + value = null; + } + + // Now add value to the row (if we're not skipping the row) + if ( r != null ) { + r[valuenr] = value; + } + } + + // none of this applies if we're skipping the row + if ( r != null ) { + // Support for trailing nullcols! + // Should be OK at allocation time, but it doesn't hurt :-) + if ( fieldnr < nrfields ) { + for ( int i = fieldnr; i < info.getInputFields().length; i++ ) { + r[shiftFields + i] = null; + } + } + + // Add the error handling fields... + int index = shiftFields + nrfields; + if ( errorCount != null ) { + r[index] = errorCount; + index++; + } + if ( errorFields != null ) { + r[index] = errorFields; + index++; + } + if ( errorText != null ) { + r[index] = errorText; + index++; + } + + // Possibly add a filename... + if ( info.includeFilename() ) { + r[index] = fname; + index++; + } + + // Possibly add a row number... + if ( info.includeRowNumber() ) { + r[index] = new Long( rowNr ); + index++; + } + + // Possibly add short filename... + if ( addShortFilename ) { + r[index] = shortFilename; + index++; + } + // Add Extension + if ( addExtension ) { + r[index] = extension; + index++; + } + // add path + if ( addPath ) { + r[index] = path; + index++; + } + // Add Size + if ( addSize ) { + r[index] = new Long( size ); + index++; + } + // add Hidden + if ( addIsHidden ) { + r[index] = hidden; + index++; + } + // Add modification date + if ( addLastModificationDate ) { + r[index] = modificationDateTime; + index++; + } + // Add Uri + if ( addUri ) { + r[index] = uri; + index++; + } + // Add RootUri + if ( addRootUri ) { + r[index] = rooturi; + index++; + } + } // End if r != null + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLineText" ), e ); + } + + if ( passThruFields != null ) { + // Simply add all fields from source files step + for ( int i = 0; i < nrPassThruFields; i++ ) { + r[i] = passThruFields[i]; + } + } + + return r; + + } + + @Override + public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { + data = (OldTextFileInputData) sdi; + meta = (OldTextFileInputMeta) smi; + Object[] r = null; + boolean retval = true; + boolean putrow = false; + + if ( first ) { // we just got started + + first = false; + + data.outputRowMeta = new RowMeta(); + RowMetaInterface[] infoStep = null; + + if ( meta.isAcceptingFilenames() ) { + // Read the files from the specified input stream... + // + data.getFiles().getFiles().clear(); + + int idx = -1; + data.rowSet = findInputRowSet( meta.getAcceptingStepName() ); + + Object[] fileRow = getRowFrom( data.rowSet ); + while ( fileRow != null ) { + RowMetaInterface prevInfoFields = data.rowSet.getRowMeta(); + if ( idx < 0 ) { + if ( meta.isPassingThruFields() ) { + data.passThruFields = new HashMap(); + infoStep = new RowMetaInterface[] { prevInfoFields }; + data.nrPassThruFields = prevInfoFields.size(); + } + idx = prevInfoFields.indexOfValue( meta.getAcceptingField() ); + if ( idx < 0 ) { + logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.UnableToFindFilenameField", meta + .getAcceptingField() ) ); + setErrors( getErrors() + 1 ); + stopAll(); + return false; + } + } + String fileValue = prevInfoFields.getString( fileRow, idx ); + try { + FileObject fileObject = KettleVFS.getFileObject( fileValue, getTransMeta() ); + data.getFiles().addFile( fileObject ); + if ( meta.isPassingThruFields() ) { + data.passThruFields.put( fileObject, fileRow ); + } + } catch ( KettleFileException e ) { + logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.UnableToCreateFileObject", fileValue ), e ); + } + + // Grab another row + fileRow = getRowFrom( data.rowSet ); + } + + if ( data.getFiles().nrOfFiles() == 0 ) { + if ( log.isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "TextFileInput.Log.Error.NoFilesSpecified" ) ); + } + setOutputDone(); + return false; + } + } + + // // get the metadata populated. Simple and easy. + meta.getFields( data.outputRowMeta, getStepname(), infoStep, null, this, repository, metaStore ); + // Create convert meta-data objects that will contain Date & Number formatters + // + data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); + + handleMissingFiles(); + + // Open the first file & read the required rows in the buffer, stop + // if it fails and not set to skip bad files... + if ( !openNextFile() ) { + if ( failAfterBadFile( null ) ) { + closeLastFile(); + setOutputDone(); + return false; + } + } + + // Count the number of repeat fields... + for ( int i = 0; i < meta.getInputFields().length; i++ ) { + if ( meta.getInputFields()[i].isRepeated() ) { + data.nr_repeats++; + } + } + } else { + if ( !data.doneReading ) { + int repeats = 1; + if ( meta.isLineWrapped() ) { + repeats = meta.getNrWraps() > 0 ? meta.getNrWraps() : repeats; + } + + if ( !data.doneWithHeader && data.headerLinesRead == 0 ) { + // We are just starting to read header lines, read them all + repeats += meta.getNrHeaderLines() + 1; + } + + // Read a number of lines... + for ( int i = 0; i < repeats && !data.doneReading; i++ ) { + tryToReadLine( true ); + } + } + } + + /* + * If the buffer is empty: open the next file. (if nothing in there, open the next, etc.) + */ + while ( data.lineBuffer.size() == 0 ) { + if ( !openNextFile() ) // Open fails: done processing unless set to skip bad files + { + if ( failAfterBadFile( null ) ) { + closeLastFile(); + setOutputDone(); // signal end to receiver(s) + return false; + } // else will continue until can open + } + } + + /* + * Take the first line available in the buffer & remove the line from the buffer + */ + OldTextFileLine textLine = data.lineBuffer.get( 0 ); + incrementLinesInput(); + lineNumberInFile++; + + data.lineBuffer.remove( 0 ); + + if ( meta.isLayoutPaged() ) { + /* + * Different rules apply: on each page: a header a number of data lines a footer + */ + if ( !data.doneWithHeader && data.pageLinesRead == 0 ) // We are reading header lines + { + if ( log.isRowLevel() ) { + logRowlevel( "P-HEADER (" + data.headerLinesRead + ") : " + textLine.line ); + } + data.headerLinesRead++; + if ( data.headerLinesRead >= meta.getNrHeaderLines() ) { + data.doneWithHeader = true; + } + } else { + // data lines or footer on a page + + if ( data.pageLinesRead < meta.getNrLinesPerPage() ) { + // See if we are dealing with wrapped lines: + if ( meta.isLineWrapped() ) { + for ( int i = 0; i < meta.getNrWraps(); i++ ) { + String extra = ""; + if ( data.lineBuffer.size() > 0 ) { + extra = data.lineBuffer.get( 0 ).line; + data.lineBuffer.remove( 0 ); + } + textLine.line += extra; + } + } + + if ( log.isRowLevel() ) { + logRowlevel( "P-DATA: " + textLine.line ); + } + // Read a normal line on a page of data. + data.pageLinesRead++; + data.lineInFile++; + long useNumber = meta.isRowNumberByFile() ? data.lineInFile : getLinesWritten() + 1; + r = + convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, data.nrPassThruFields, + data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, data.separator, data.enclosure, + data.escapeCharacter, data.dataErrorLineHandler, data.addShortFilename, data.addExtension, + data.addPath, data.addSize, data.addIsHidden, data.addLastModificationDate, data.addUri, + data.addRootUri, data.shortFilename, data.path, data.hidden, data.lastModificationDateTime, + data.uriName, data.rootUriName, data.extension, data.size ); + if ( r != null ) { + putrow = true; + } + + // Possible fix for bug PDI-1121 - paged layout header and line count off by 1 + // We need to reset these BEFORE the next header line is read, so that it + // is treated as a header ... obviously, only if there is no footer, and we are + // done reading data. + if ( !meta.hasFooter() && ( data.pageLinesRead == meta.getNrLinesPerPage() ) ) { + /* + * OK, we are done reading the footer lines, start again on 'next page' with the header + */ + data.doneWithHeader = false; + data.headerLinesRead = 0; + data.pageLinesRead = 0; + data.footerLinesRead = 0; + if ( log.isRowLevel() ) { + logRowlevel( "RESTART PAGE" ); + } + } + } else { + // done reading the data lines, skip the footer lines + + if ( meta.hasFooter() && data.footerLinesRead < meta.getNrFooterLines() ) { + if ( log.isRowLevel() ) { + logRowlevel( "P-FOOTER: " + textLine.line ); + } + data.footerLinesRead++; + } + + if ( !meta.hasFooter() || data.footerLinesRead >= meta.getNrFooterLines() ) { + /* + * OK, we are done reading the footer lines, start again on 'next page' with the header + */ + data.doneWithHeader = false; + data.headerLinesRead = 0; + data.pageLinesRead = 0; + data.footerLinesRead = 0; + if ( log.isRowLevel() ) { + logRowlevel( "RESTART PAGE" ); + } + } + } + } + } else { + // A normal data line, can also be a header or a footer line + + if ( !data.doneWithHeader ) { // We are reading header lines + + data.headerLinesRead++; + if ( data.headerLinesRead >= meta.getNrHeaderLines() ) { + data.doneWithHeader = true; + } + } else { + /* + * IF we are done reading and we have a footer AND the number of lines in the buffer is smaller then the number + * of footer lines THEN we can remove the remaining rows from the buffer: they are all footer rows. + */ + if ( data.doneReading && meta.hasFooter() && data.lineBuffer.size() < meta.getNrFooterLines() ) { + data.lineBuffer.clear(); + } else { + // Not yet a footer line: it's a normal data line. + + // See if we are dealing with wrapped lines: + if ( meta.isLineWrapped() ) { + for ( int i = 0; i < meta.getNrWraps(); i++ ) { + String extra = ""; + if ( data.lineBuffer.size() > 0 ) { + extra = data.lineBuffer.get( 0 ).line; + data.lineBuffer.remove( 0 ); + } else { + tryToReadLine( true ); + if ( !data.lineBuffer.isEmpty() ) { + extra = data.lineBuffer.remove( 0 ).line; + } + } + textLine.line += extra; + } + } + if ( data.filePlayList.isProcessingNeeded( textLine.file, textLine.lineNumber, + AbstractFileErrorHandler.NO_PARTS ) ) { + data.lineInFile++; + long useNumber = meta.isRowNumberByFile() ? data.lineInFile : getLinesWritten() + 1; + r = + convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, data.nrPassThruFields, + data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, data.separator, data.enclosure, + data.escapeCharacter, data.dataErrorLineHandler, data.addShortFilename, data.addExtension, + data.addPath, data.addSize, data.addIsHidden, data.addLastModificationDate, data.addUri, + data.addRootUri, data.shortFilename, data.path, data.hidden, data.lastModificationDateTime, + data.uriName, data.rootUriName, data.extension, data.size ); + if ( r != null ) { + if ( log.isRowLevel() ) { + logRowlevel( "Found data row: " + data.outputRowMeta.getString( r ) ); + } + putrow = true; + } + } else { + putrow = false; + } + } + } + } + + if ( putrow && r != null ) { + // See if the previous values need to be repeated! + if ( data.nr_repeats > 0 ) { + if ( data.previous_row == null ) { // First invocation... + + data.previous_row = data.outputRowMeta.cloneRow( r ); + } else { + // int repnr = 0; + for ( int i = 0; i < meta.getInputFields().length; i++ ) { + if ( meta.getInputFields()[i].isRepeated() ) { + if ( r[i] == null ) { + // if it is empty: take the previous value! + + r[i] = data.previous_row[i]; + } else { + // not empty: change the previous_row entry! + + data.previous_row[i] = r[i]; + } + // repnr++; + } + } + } + } + + if ( log.isRowLevel() ) { + logRowlevel( "Putting row: " + data.outputRowMeta.getString( r ) ); + } + putRow( data.outputRowMeta, r ); + + if ( getLinesInput() >= meta.getRowLimit() && meta.getRowLimit() > 0 ) { + closeLastFile(); + setOutputDone(); // signal end to receiver(s) + return false; + } + } + + if ( checkFeedback( getLinesInput() ) ) { + if ( log.isBasic() ) { + logBasic( "linenr " + getLinesInput() ); + } + } + + return retval; + } + + /** + * + * @param errorMsg + * Message to send to rejected row if enabled + * @return If should stop processing after having problems with a file + */ + private boolean failAfterBadFile( String errorMsg ) { + + if ( getStepMeta().isDoingErrorHandling() && data.filename != null + && !data.rejectedFiles.containsKey( data.filename ) ) { + data.rejectedFiles.put( data.filename, true ); + rejectCurrentFile( errorMsg ); + } + + return !meta.isErrorIgnored() || !meta.isSkipBadFiles() || data.isLastFile; + } + + /** + * Send file name and/or error message to error output + * + * @param errorMsg + * Message to send to rejected row if enabled + */ + private void rejectCurrentFile( String errorMsg ) { + if ( StringUtils.isNotBlank( meta.getFileErrorField() ) + || StringUtils.isNotBlank( meta.getFileErrorMessageField() ) ) { + RowMetaInterface rowMeta = getInputRowMeta(); + if ( rowMeta == null ) { + rowMeta = new RowMeta(); + } + + int errorFileIndex = + ( StringUtils.isBlank( meta.getFileErrorField() ) ) ? -1 : addValueMeta( rowMeta, this + .environmentSubstitute( meta.getFileErrorField() ) ); + + int errorMessageIndex = + StringUtils.isBlank( meta.getFileErrorMessageField() ) ? -1 : addValueMeta( rowMeta, this + .environmentSubstitute( meta.getFileErrorMessageField() ) ); + + try { + Object[] rowData = getRow(); + if ( rowData == null ) { + rowData = RowDataUtil.allocateRowData( rowMeta.size() ); + } + + if ( errorFileIndex >= 0 ) { + rowData[errorFileIndex] = data.filename; + } + if ( errorMessageIndex >= 0 ) { + rowData[errorMessageIndex] = errorMsg; + } + + putError( rowMeta, rowData, getErrors(), data.filename, null, "ERROR_CODE" ); + } catch ( Exception e ) { + logError( "Error sending error row", e ); + } + } + } + + /** + * Adds String value meta with given name if not present and returns index + * + * @param rowMeta + * @param fieldName + * @return Index in row meta of value meta with fieldName + */ + private int addValueMeta( RowMetaInterface rowMeta, String fieldName ) { + ValueMetaInterface valueMeta = new ValueMeta( fieldName, ValueMetaInterface.TYPE_STRING ); + valueMeta.setOrigin( getStepname() ); + // add if doesn't exist + int index = -1; + if ( !rowMeta.exists( valueMeta ) ) { + index = rowMeta.size(); + rowMeta.addValueMeta( valueMeta ); + } else { + index = rowMeta.indexOfValue( fieldName ); + } + return index; + } + + /** + * Check if the line should be taken. + * + * @param line + * @param isFilterLastLine + * (dummy input param, only set when return value is false) + * @return true when the line should be taken (when false, isFilterLastLine will be set) + */ + private boolean checkFilterRow( String line, boolean isFilterLastLine ) { + boolean filterOK = true; + + // check for noEmptyLines + if ( meta.noEmptyLines() && line.length() == 0 ) { + filterOK = false; + } else { + // check the filters + filterOK = data.filterProcessor.doFilters( line ); + if ( !filterOK ) { + if ( data.filterProcessor.isStopProcessing() ) { + data.doneReading = true; + } + } + } + + return filterOK; + } + + private void handleMissingFiles() throws KettleException { + List nonExistantFiles = data.getFiles().getNonExistantFiles(); + + if ( nonExistantFiles.size() != 0 ) { + String message = FileInputList.getRequiredFilesDescription( nonExistantFiles ); + if ( log.isBasic() ) { + log.logBasic( "Required files", "WARNING: Missing " + message ); + } + if ( meta.isErrorIgnored() ) { + for ( FileObject fileObject : nonExistantFiles ) { + data.dataErrorLineHandler.handleNonExistantFile( fileObject ); + } + } else { + throw new KettleException( "Following required files are missing: " + message ); + } + } + + List nonAccessibleFiles = data.getFiles().getNonAccessibleFiles(); + if ( nonAccessibleFiles.size() != 0 ) { + String message = FileInputList.getRequiredFilesDescription( nonAccessibleFiles ); + if ( log.isBasic() ) { + log.logBasic( "Required files", "WARNING: Not accessible " + message ); + } + if ( meta.isErrorIgnored() ) { + for ( FileObject fileObject : nonAccessibleFiles ) { + data.dataErrorLineHandler.handleNonAccessibleFile( fileObject ); + } + } else { + throw new KettleException( "Following required files are not accessible: " + message ); + } + } + } + + private boolean closeLastFile() { + try { + // Close previous file! + if ( data.filename != null ) { + // Increment the lines updated to reflect another file has been finished. + // This allows us to give a state of progress in the run time metrics + incrementLinesUpdated(); + /* + * } else if ( sFileCompression != null && sFileCompression.equals( "Snappy" ) && data.sis != null ) { + * data.sis.close(); } + */ + data.in.close(); + data.isr.close(); + data.filename = null; // send it down the next time. + if ( data.file != null ) { + data.file.close(); + data.file = null; + } + } + data.dataErrorLineHandler.close(); + } catch ( Exception e ) { + String errorMsg = "Couldn't close file : " + data.file.getName().getFriendlyURI() + " --> " + e.toString(); + logError( errorMsg ); + if ( failAfterBadFile( errorMsg ) ) { // ( !meta.isSkipBadFiles() || data.isLastFile ){ + stopAll(); + } + setErrors( getErrors() + 1 ); + return false; + } // finally { + // This is for bug #5797 : it tries to assure that the file handle + // is actually freed/garbarge collected. + // XXX deinspanjer 2009-07-07: I'm stubbing this out. The bug was ancient and it is worth reevaluating + // to avoid the performance hit of a System GC on every file close + // System.gc(); + // } + + return !data.isLastFile; + } + + private boolean openNextFile() { + + try { + lineNumberInFile = 0; + if ( !closeLastFile() && failAfterBadFile( null ) ) { + return false; // (!meta.isSkipBadFiles() || data.isLastFile) ) return false; + } + + if ( data.getFiles().nrOfFiles() == 0 ) { + return false; + } + + // Is this the last file? + data.isLastFile = ( data.filenr == data.getFiles().nrOfFiles() - 1 ); + data.file = data.getFiles().getFile( data.filenr ); + data.filename = KettleVFS.getFilename( data.file ); + + // Move file pointer ahead! + data.filenr++; + + // Add additional fields? + if ( data.addShortFilename ) { + data.shortFilename = data.file.getName().getBaseName(); + } + if ( data.addPath ) { + data.path = KettleVFS.getFilename( data.file.getParent() ); + } + if ( data.addIsHidden ) { + data.hidden = data.file.isHidden(); + } + if ( data.addExtension ) { + data.extension = data.file.getName().getExtension(); + } + if ( data.addLastModificationDate ) { + data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); + } + if ( data.addUri ) { + data.uriName = data.file.getName().getURI(); + } + if ( data.addRootUri ) { + data.rootUriName = data.file.getName().getRootURI(); + } + if ( data.addSize ) { + data.size = new Long( data.file.getContent().getSize() ); + } + data.lineInFile = 0; + if ( meta.isPassingThruFields() ) { + data.currentPassThruFieldsRow = data.passThruFields.get( data.file ); + } + + // Add this files to the result of this transformation. + // + if ( meta.isAddResultFile() ) { + ResultFile resultFile = + new ResultFile( ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), toString() ); + resultFile.setComment( "File was read by an Text File input step" ); + addResultFile( resultFile ); + } + if ( log.isBasic() ) { + logBasic( "Opening file: " + data.file.getName().getFriendlyURI() ); + } + + CompressionProvider provider = + CompressionProviderFactory.getInstance().getCompressionProviderByName( meta.getFileCompression() ); + + data.in = provider.createInputStream( KettleVFS.getInputStream( data.file ) ); + data.dataErrorLineHandler.handleFile( data.file ); + data.in.nextEntry(); + + if ( log.isDetailed() ) { + logDetailed( "This is a compressed file being handled by the " + provider.getName() + " provider" ); + } + + if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { + data.isr = + new InputStreamReader( new BufferedInputStream( data.in, BUFFER_SIZE_INPUT_STREAM ), meta.getEncoding() ); + } else { + data.isr = new InputStreamReader( new BufferedInputStream( data.in, BUFFER_SIZE_INPUT_STREAM ) ); + } + + String encoding = data.isr.getEncoding(); + data.encodingType = OldEncodingType.guessEncodingType( encoding ); + + // ///////////////////////////////////////////////////////////////////////////// + // Read the first lines... + + /* + * Keep track of the status of the file: are there any lines left to read? + */ + data.doneReading = false; + + /* + * OK, read a number of lines in the buffer: The header rows The nr rows in the page : optional The footer rows + */ + int bufferSize = 1; + bufferSize += meta.hasHeader() ? meta.getNrHeaderLines() : 0; + bufferSize += meta.isLayoutPaged() + ? meta.getNrLinesPerPage() * ( Math.max( 0, meta.getNrWraps() ) + 1 ) + : Math.max( 0, meta.getNrWraps() ); // it helps when we have wrapped input w/o header + + bufferSize += meta.hasFooter() ? meta.getNrFooterLines() : 0; + + // See if we need to skip the document header lines... + if ( meta.isLayoutPaged() ) { + for ( int i = 0; i < meta.getNrLinesDocHeader(); i++ ) { + // Just skip these... + getLine( log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); // header and + // footer: not + // wrapped + lineNumberInFile++; + } + } + + for ( int i = 0; i < bufferSize && !data.doneReading; i++ ) { + boolean wasNotFiltered = tryToReadLine( !meta.hasHeader() || i >= meta.getNrHeaderLines() ); + if ( !wasNotFiltered ) { + // grab another line, this one got filtered + bufferSize++; + } + } + + // Reset counters etc. + data.headerLinesRead = 0; + data.footerLinesRead = 0; + data.pageLinesRead = 0; + + // Set a flags + data.doneWithHeader = !meta.hasHeader(); + } catch ( Exception e ) { + String errorMsg = + "Couldn't open file #" + data.filenr + " : " + data.file.getName().getFriendlyURI() + " --> " + e.toString(); + logError( errorMsg ); + if ( failAfterBadFile( errorMsg ) ) { // !meta.isSkipBadFiles()) stopAll(); + stopAll(); + } + setErrors( getErrors() + 1 ); + return false; + } + return true; + } + + private boolean tryToReadLine( boolean applyFilter ) throws KettleFileException { + String line; + line = getLine( log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); + if ( line != null ) { + // when there is no header, check the filter for the first line + if ( applyFilter ) { + // Filter row? + boolean isFilterLastLine = false; + boolean filterOK = checkFilterRow( line, isFilterLastLine ); + if ( filterOK ) { + data.lineBuffer.add( new OldTextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the + // line buffer... + } else { + return false; + } + } else { // don't checkFilterRow + + if ( !meta.noEmptyLines() || line.length() != 0 ) { + data.lineBuffer.add( new OldTextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the line + // buffer... + } + } + } else { + data.doneReading = true; + } + return true; + } + + @Override + public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { + meta = (OldTextFileInputMeta) smi; + data = (OldTextFileInputData) sdi; + + if ( super.init( smi, sdi ) ) { + initErrorHandling(); + initReplayFactory(); + + data.setFiles( meta.getTextFileList( this ) ); + data.filterProcessor = new OldTextFileFilterProcessor( meta.getFilter() ); + + // If there are missing files, + // fail if we don't ignore errors + // + Result previousResult = getTrans().getPreviousResult(); + Map resultFiles = ( previousResult != null ) ? previousResult.getResultFiles() : null; + + if ( ( previousResult == null || resultFiles == null || resultFiles.size() == 0 ) + && data.getFiles().nrOfMissingFiles() > 0 && !meta.isAcceptingFilenames() && !meta.isErrorIgnored() ) { + logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.NoFilesSpecified" ) ); + return false; + } + + String clusterSize = getVariable( Const.INTERNAL_VARIABLE_CLUSTER_SIZE ); + if ( !Const.isEmpty( clusterSize ) && Integer.valueOf( clusterSize ) > 1 ) { + // TODO: add metadata to configure this. + String nr = getVariable( Const.INTERNAL_VARIABLE_SLAVE_SERVER_NUMBER ); + if ( log.isDetailed() ) { + logDetailed( "Running on slave server #" + nr + + " : assuming that each slave reads a dedicated part of the same file(s)." ); + } + } + + // If no nullif field is supplied, take the default. + // String null_value = nullif; + // if (null_value == null) + // { + // // value=""; + // } + // String null_cmp = Const.rightPad(new StringBuilder(null_value), pol.length()); + + // calculate the file format type in advance so we can use a switch + data.fileFormatType = meta.getFileFormatTypeNr(); + + // calculate the file type in advance CSV or Fixed? + data.fileType = meta.getFileTypeNr(); + + // Handle the possibility of a variable substitution + data.separator = environmentSubstitute( meta.getSeparator() ); + data.enclosure = environmentSubstitute( meta.getEnclosure() ); + data.escapeCharacter = environmentSubstitute( meta.getEscapeCharacter() ); + + // Add additional fields + if ( !Const.isEmpty( meta.getShortFileNameField() ) ) { + data.addShortFilename = true; + } + if ( !Const.isEmpty( meta.getPathField() ) ) { + data.addPath = true; + } + if ( !Const.isEmpty( meta.getExtensionField() ) ) { + data.addExtension = true; + } + if ( !Const.isEmpty( meta.getSizeField() ) ) { + data.addSize = true; + } + if ( !Const.isEmpty( meta.isHiddenField() ) ) { + data.addIsHidden = true; + } + if ( !Const.isEmpty( meta.getLastModificationDateField() ) ) { + data.addLastModificationDate = true; + } + if ( !Const.isEmpty( meta.getUriField() ) ) { + data.addUri = true; + } + if ( !Const.isEmpty( meta.getRootUriField() ) ) { + data.addRootUri = true; + } + return true; + } + return false; + } + + private void initReplayFactory() { + Date replayDate = getTrans().getReplayDate(); + if ( replayDate == null ) { + data.filePlayList = FilePlayListAll.INSTANCE; + } else { + data.filePlayList = + new FilePlayListReplay( replayDate, meta.getLineNumberFilesDestinationDirectory(), meta + .getLineNumberFilesExtension(), meta.getErrorFilesDestinationDirectory(), meta + .getErrorLineFilesExtension(), meta.getEncoding() ); + } + } + + private void initErrorHandling() { + List dataErrorLineHandlers = new ArrayList( 2 ); + if ( meta.getLineNumberFilesDestinationDirectory() != null ) { + dataErrorLineHandlers + .add( new FileErrorHandlerContentLineNumber( getTrans().getCurrentDate(), environmentSubstitute( meta + .getLineNumberFilesDestinationDirectory() ), meta.getLineNumberFilesExtension(), meta.getEncoding(), this ) ); + } + if ( meta.getErrorFilesDestinationDirectory() != null ) { + dataErrorLineHandlers.add( new FileErrorHandlerMissingFiles( getTrans().getCurrentDate(), environmentSubstitute( + meta.getErrorFilesDestinationDirectory() ), meta.getErrorLineFilesExtension(), meta.getEncoding(), this ) ); + } + data.dataErrorLineHandler = new CompositeFileErrorHandler( dataErrorLineHandlers ); + } + + @Override + public void dispose( StepMetaInterface smi, StepDataInterface sdi ) { + meta = (OldTextFileInputMeta) smi; + data = (OldTextFileInputData) sdi; + + if ( data.file != null ) { + try { + data.file.close(); + data.file = null; + } catch ( Exception e ) { + log.logError( "Error closing file", e ); + } + } + if ( data.in != null ) { + BaseStep.closeQuietly( data.in ); + data.in = null; + } + super.dispose( smi, sdi ); + } + + public boolean isWaitingForData() { + return true; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputData.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputData.java new file mode 100644 index 000000000000..172401e9e40b --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputData.java @@ -0,0 +1,164 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.io.InputStreamReader; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.RowSet; +import org.pentaho.di.core.compress.CompressionInputStream; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.playlist.FilePlayList; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.trans.step.BaseStepData; +import org.pentaho.di.trans.step.StepDataInterface; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; + +/** + * @author Matt + * @since 22-jan-2005 + */ +public class OldTextFileInputData extends BaseStepData implements StepDataInterface { + + public List lineBuffer; + + public Object[] previous_row; + + public int nr_repeats; + + public int nrLinesOnPage; + + private FileInputList files; + + public HashMap passThruFields; + + public Object[] currentPassThruFieldsRow; + + public int nrPassThruFields; + + public boolean isLastFile; + + public String filename; + + public int lineInFile; + + public FileObject file; + + public int filenr; + + public CompressionInputStream in; + + public InputStreamReader isr; + + public boolean doneReading; + + public int headerLinesRead; + + public int footerLinesRead; + + public int pageLinesRead; + + public boolean doneWithHeader; + + public FileErrorHandler dataErrorLineHandler; + + public FilePlayList filePlayList; + + public OldTextFileFilterProcessor filterProcessor; + + public RowMetaInterface outputRowMeta; + + public StringBuilder lineStringBuilder; + + public int fileFormatType; + + public int fileType; + + public RowMetaInterface convertRowMeta; + + public RowSet rowSet; + + /** + * The separator (delimiter) + */ + public String separator; + + public String enclosure; + + public String escapeCharacter; + + public boolean addShortFilename; + public boolean addExtension; + public boolean addPath; + public boolean addSize; + public boolean addIsHidden; + public boolean addLastModificationDate; + public boolean addUri; + public boolean addRootUri; + + public String shortFilename; + public String path; + public String extension; + public boolean hidden; + public Date lastModificationDateTime; + public String uriName; + public String rootUriName; + public long size; + + public OldEncodingType encodingType; + + public Map rejectedFiles; + + public OldTextFileInputData() { + super(); + + // linked list is better, as usually .remove(0) is applied to this list + lineBuffer = new LinkedList(); + + nr_repeats = 0; + previous_row = null; + filenr = 0; + + nrLinesOnPage = 0; + + in = null; + + filterProcessor = null; + lineStringBuilder = new StringBuilder( 256 ); + + rejectedFiles = new HashMap(); + } + + public FileInputList getFiles() { + return files; + } + + public void setFiles( FileInputList files ) { + this.files = files; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMeta.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMeta.java new file mode 100644 index 000000000000..2d0b93389d59 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMeta.java @@ -0,0 +1,2068 @@ +// CHECKSTYLE:FileLength:OFF +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.CheckResult; +import org.pentaho.di.core.CheckResultInterface; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.exception.KettleXMLException; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaFactory; +import org.pentaho.di.core.util.EnvUtil; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.core.xml.XMLHandler; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.repository.ObjectId; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.resource.ResourceDefinition; +import org.pentaho.di.resource.ResourceEntry; +import org.pentaho.di.resource.ResourceEntry.ResourceType; +import org.pentaho.di.resource.ResourceNamingInterface; +import org.pentaho.di.resource.ResourceReference; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStepMeta; +import org.pentaho.di.trans.step.StepDataInterface; +import org.pentaho.di.trans.step.StepInterface; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInjectionInterface; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.textfileinput.InputFileMetaInterface; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.metastore.api.IMetaStore; +import org.w3c.dom.Node; + +import com.google.common.annotations.VisibleForTesting; + +public class OldTextFileInputMeta extends BaseStepMeta implements StepMetaInterface, InputFileMetaInterface { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + public static final String[] RequiredFilesDesc = new String[] { + BaseMessages.getString( PKG, "System.Combo.No" ), BaseMessages.getString( PKG, "System.Combo.Yes" ) }; + public static final String[] RequiredFilesCode = new String[] { "N", "Y" }; + + private static final String NO = "N"; + + private static final String YES = "Y"; + + private static final String STRING_BASE64_PREFIX = "Base64: "; + + public static final int FILE_FORMAT_DOS = 0; + public static final int FILE_FORMAT_UNIX = 1; + public static final int FILE_FORMAT_MIXED = 2; + + public static final int FILE_TYPE_CSV = 0; + public static final int FILE_TYPE_FIXED = 1; + + /** Array of filenames */ + private String[] fileName; + + /** Wildcard or filemask (regular expression) */ + private String[] fileMask; + + /** Wildcard or filemask to exclude (regular expression) */ + private String[] excludeFileMask; + + /** Array of boolean values as string, indicating if a file is required. */ + private String[] fileRequired; + + /** Type of file: CSV or fixed */ + private String fileType; + + /** String used to separated field (;) */ + private String separator; + + /** String used to enclose separated fields (") */ + private String enclosure; + + /** Escape character used to escape the enclosure String (\) */ + private String escapeCharacter; + + /** Switch to allow breaks (CR/LF) in Enclosures */ + private boolean breakInEnclosureAllowed; + + /** Flag indicating that the file contains one header line that should be skipped. */ + private boolean header; + + /** The number of header lines, defaults to 1 */ + private int nrHeaderLines; + + /** Flag indicating that the file contains one footer line that should be skipped. */ + private boolean footer; + + /** The number of footer lines, defaults to 1 */ + private int nrFooterLines; + + /** Flag indicating that a single line is wrapped onto one or more lines in the text file. */ + private boolean lineWrapped; + + /** The number of times the line wrapped */ + private int nrWraps; + + /** Flag indicating that the text-file has a paged layout. */ + private boolean layoutPaged; + + /** The number of lines in the document header */ + private int nrLinesDocHeader; + + /** The number of lines to read per page */ + private int nrLinesPerPage; + + /** Type of compression being used */ + private String fileCompression; + + /** Flag indicating that we should skip all empty lines */ + private boolean noEmptyLines; + + /** Flag indicating that we should include the filename in the output */ + private boolean includeFilename; + + /** The name of the field in the output containing the filename */ + private String filenameField; + + /** Flag indicating that a row number field should be included in the output */ + private boolean includeRowNumber; + + /** Flag indicating row number is per file */ + private boolean rowNumberByFile; + + /** The name of the field in the output containing the row number */ + private String rowNumberField; + + /** The file format: DOS or UNIX or mixed */ + private String fileFormat; + + /** The maximum number or lines to read */ + private long rowLimit; + + /** The fields to import... */ + private TextFileInputField[] inputFields; + + /** Array of boolean values as string, indicating if we need to fetch sub folders. */ + private String[] includeSubFolders; + + /** The filters to use... */ + private OldTextFileFilter[] filter; + + /** The encoding to use for reading: null or empty string means system default encoding */ + private String encoding; + + /** Ignore error : turn into warnings */ + private boolean errorIgnored; + + /** The name of the field that will contain the number of errors in the row */ + private String errorCountField; + + /** The name of the field that will contain the names of the fields that generated errors, separated by , */ + private String errorFieldsField; + + /** The name of the field that will contain the error texts, separated by CR */ + private String errorTextField; + + /** The directory that will contain warning files */ + private String warningFilesDestinationDirectory; + + /** The extension of warning files */ + private String warningFilesExtension; + + /** The directory that will contain error files */ + private String errorFilesDestinationDirectory; + + /** The extension of error files */ + private String errorFilesExtension; + + /** The directory that will contain line number files */ + private String lineNumberFilesDestinationDirectory; + + /** The extension of line number files */ + private String lineNumberFilesExtension; + + /** Indicate whether or not we want to date fields strictly according to the format or lenient */ + private boolean dateFormatLenient; + + /** Specifies the Locale of the Date format, null means the default */ + private Locale dateFormatLocale; + + /** If error line are skipped, you can replay without introducing doubles. */ + private boolean errorLineSkipped; + + /** Are we accepting filenames in input rows? */ + private boolean acceptingFilenames; + + /** If receiving input rows, should we pass through existing fields? */ + private boolean passingThruFields; + + /** The field in which the filename is placed */ + private String acceptingField; + + /** The stepname to accept filenames from */ + private String acceptingStepName; + + /** The step to accept filenames from */ + private StepMeta acceptingStep; + + /** The add filenames to result filenames flag */ + private boolean isaddresult; + + /** Additional fields **/ + private String shortFileFieldName; + private String pathFieldName; + private String hiddenFieldName; + private String lastModificationTimeFieldName; + private String uriNameFieldName; + private String rootUriNameFieldName; + private String extensionFieldName; + private String sizeFieldName; + + private boolean skipBadFiles; + private String fileErrorField; + private String fileErrorMessageField; + + /** + * @return Returns the shortFileFieldName. + */ + public String getShortFileNameField() { + return shortFileFieldName; + } + + /** + * @param field + * The shortFileFieldName to set. + */ + public void setShortFileNameField( String field ) { + shortFileFieldName = field; + } + + /** + * @return Returns the pathFieldName. + */ + public String getPathField() { + return pathFieldName; + } + + /** + * @param field + * The pathFieldName to set. + */ + public void setPathField( String field ) { + this.pathFieldName = field; + } + + /** + * @return Returns the hiddenFieldName. + */ + public String isHiddenField() { + return hiddenFieldName; + } + + /** + * @param field + * The hiddenFieldName to set. + */ + public void setIsHiddenField( String field ) { + hiddenFieldName = field; + } + + /** + * @return Returns the lastModificationTimeFieldName. + */ + public String getLastModificationDateField() { + return lastModificationTimeFieldName; + } + + /** + * @param field + * The lastModificationTimeFieldName to set. + */ + public void setLastModificationDateField( String field ) { + lastModificationTimeFieldName = field; + } + + /** + * @return Returns the uriNameFieldName. + */ + public String getUriField() { + return uriNameFieldName; + } + + /** + * @param field + * The uriNameFieldName to set. + */ + public void setUriField( String field ) { + uriNameFieldName = field; + } + + /** + * @return Returns the uriNameFieldName. + */ + public String getRootUriField() { + return rootUriNameFieldName; + } + + /** + * @param field + * The rootUriNameFieldName to set. + */ + public void setRootUriField( String field ) { + rootUriNameFieldName = field; + } + + /** + * @return Returns the extensionFieldName. + */ + public String getExtensionField() { + return extensionFieldName; + } + + /** + * @param field + * The extensionFieldName to set. + */ + public void setExtensionField( String field ) { + extensionFieldName = field; + } + + /** + * @return Returns the sizeFieldName. + */ + public String getSizeField() { + return sizeFieldName; + } + + /** + * @param field + * The sizeFieldName to set. + */ + public void setSizeField( String field ) { + sizeFieldName = field; + } + + /** + * + * @return If should continue processing after failing to open a file + */ + public boolean isSkipBadFiles() { + return skipBadFiles; + } + + /** + * + * @param value + * If should continue processing after failing to open a file + */ + public void setSkipBadFiles( boolean value ) { + skipBadFiles = value; + } + + public String getFileErrorField() { + return fileErrorField; + } + + public void setFileErrorField( String field ) { + fileErrorField = field; + } + + public String getFileErrorMessageField() { + return fileErrorMessageField; + } + + public void setFileErrorMessageField( String field ) { + fileErrorMessageField = field; + } + + /** + * @return Returns the encoding. + */ + public String getEncoding() { + return encoding; + } + + /** + * @param encoding + * The encoding to set. + */ + public void setEncoding( String encoding ) { + this.encoding = encoding; + } + + public OldTextFileInputMeta() { + super(); // allocate BaseStepMeta + } + + /** + * @return Returns the input fields. + */ + public TextFileInputField[] getInputFields() { + return inputFields; + } + + /** + * @param inputFields + * The input fields to set. + */ + public void setInputFields( TextFileInputField[] inputFields ) { + this.inputFields = inputFields; + } + + /** + * @return Returns the enclosure. + */ + public String getEnclosure() { + return enclosure; + } + + /** + * @param enclosure + * The enclosure to set. + */ + public void setEnclosure( String enclosure ) { + this.enclosure = enclosure; + } + + /** + * @return Returns the breakInEnclosureAllowed. + */ + public boolean isBreakInEnclosureAllowed() { + return breakInEnclosureAllowed; + } + + /** + * @param breakInEnclosureAllowed + * The breakInEnclosureAllowed to set. + */ + public void setBreakInEnclosureAllowed( boolean breakInEnclosureAllowed ) { + this.breakInEnclosureAllowed = breakInEnclosureAllowed; + } + + /** + * @return Returns the excludeFileMask. + */ + public String[] getExludeFileMask() { + return excludeFileMask; + } + + /** + * @param excludeFileMask + * The excludeFileMask to set. + */ + public void setExcludeFileMask( String[] excludeFileMask ) { + this.excludeFileMask = excludeFileMask; + } + + /** + * @return Returns the fileFormat. + */ + public String getFileFormat() { + return fileFormat; + } + + /** + * @param fileFormat + * The fileFormat to set. + */ + public void setFileFormat( String fileFormat ) { + this.fileFormat = fileFormat; + } + + /** + * @return Returns the fileMask. + */ + public String[] getFileMask() { + return fileMask; + } + + /** + * @return Returns the fileRequired. + */ + public String[] getFileRequired() { + return fileRequired; + } + + /** + * @param fileMask + * The fileMask to set. + */ + public void setFileMask( String[] fileMask ) { + this.fileMask = fileMask; + } + + /** + * @param fileRequired + * The fileRequired to set. + */ + public void setFileRequired( String[] fileRequiredin ) { + for ( int i = 0; i < fileRequiredin.length; i++ ) { + this.fileRequired[i] = getRequiredFilesCode( fileRequiredin[i] ); + } + } + + public String[] getIncludeSubFolders() { + return includeSubFolders; + } + + public void setIncludeSubFolders( String[] includeSubFoldersin ) { + for ( int i = 0; i < includeSubFoldersin.length; i++ ) { + this.includeSubFolders[i] = getRequiredFilesCode( includeSubFoldersin[i] ); + } + } + + public String getRequiredFilesCode( String tt ) { + if ( tt == null ) { + return RequiredFilesCode[0]; + } + if ( tt.equals( RequiredFilesDesc[1] ) ) { + return RequiredFilesCode[1]; + } else { + return RequiredFilesCode[0]; + } + } + + /** + * @return Returns the fileName. + */ + public String[] getFileName() { + return fileName; + } + + /** + * @param fileName + * The fileName to set. + */ + public void setFileName( String[] fileName ) { + this.fileName = fileName; + } + + /** + * @return Returns the filenameField. + */ + public String getFilenameField() { + return filenameField; + } + + /** + * @param filenameField + * The filenameField to set. + */ + public void setFilenameField( String filenameField ) { + this.filenameField = filenameField; + } + + /** + * @return Returns the fileType. + */ + public String getFileType() { + return fileType; + } + + /** + * @param fileType + * The fileType to set. + */ + public void setFileType( String fileType ) { + this.fileType = fileType; + } + + /** + * @return The array of filters for the metadata of this text file input step. + */ + public OldTextFileFilter[] getFilter() { + return filter; + } + + /** + * @param filter + * The array of filters to use + */ + public void setFilter( OldTextFileFilter[] filter ) { + this.filter = filter; + } + + /** + * @return Returns the footer. + */ + public boolean hasFooter() { + return footer; + } + + /** + * @param footer + * The footer to set. + */ + public void setFooter( boolean footer ) { + this.footer = footer; + } + + /** + * @return Returns the header. + */ + public boolean hasHeader() { + return header; + } + + /** + * @param header + * The header to set. + */ + public void setHeader( boolean header ) { + this.header = header; + } + + /** + * @return Returns the includeFilename. + */ + public boolean includeFilename() { + return includeFilename; + } + + /** + * @param includeFilename + * The includeFilename to set. + */ + public void setIncludeFilename( boolean includeFilename ) { + this.includeFilename = includeFilename; + } + + /** + * @return Returns the includeRowNumber. + */ + public boolean includeRowNumber() { + return includeRowNumber; + } + + /** + * @param includeRowNumber + * The includeRowNumber to set. + */ + public void setIncludeRowNumber( boolean includeRowNumber ) { + this.includeRowNumber = includeRowNumber; + } + + /** + * true if row number reset for each file + * + * @return rowNumberByFile + */ + public boolean isRowNumberByFile() { + return rowNumberByFile; + } + + /** + * @param rowNumberByFile + * True if row number field is reset for each file + */ + public void setRowNumberByFile( boolean rowNumberByFile ) { + this.rowNumberByFile = rowNumberByFile; + } + + /** + * @return Returns the noEmptyLines. + */ + public boolean noEmptyLines() { + return noEmptyLines; + } + + /** + * @param noEmptyLines + * The noEmptyLines to set. + */ + public void setNoEmptyLines( boolean noEmptyLines ) { + this.noEmptyLines = noEmptyLines; + } + + /** + * @return Returns the rowLimit. + */ + public long getRowLimit() { + return rowLimit; + } + + /** + * @param rowLimit + * The rowLimit to set. + */ + public void setRowLimit( long rowLimit ) { + this.rowLimit = rowLimit; + } + + /** + * @return Returns the rowNumberField. + */ + public String getRowNumberField() { + return rowNumberField; + } + + /** + * @param rowNumberField + * The rowNumberField to set. + */ + public void setRowNumberField( String rowNumberField ) { + this.rowNumberField = rowNumberField; + } + + /** + * @return Returns the separator. + */ + public String getSeparator() { + return separator; + } + + /** + * @param separator + * The separator to set. + */ + public void setSeparator( String separator ) { + this.separator = separator; + } + + /** + * @return Returns the type of compression used + */ + public String getFileCompression() { + return fileCompression; + } + + /** + * @param fileCompression + * Sets the compression type + */ + public void setFileCompression( String fileCompression ) { + this.fileCompression = fileCompression; + } + + public void loadXML( Node stepnode, List databases, IMetaStore metaStore ) throws KettleXMLException { + try { + acceptingFilenames = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "accept_filenames" ) ); + passingThruFields = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "passing_through_fields" ) ); + acceptingField = XMLHandler.getTagValue( stepnode, "accept_field" ); + acceptingStepName = XMLHandler.getTagValue( stepnode, "accept_stepname" ); + + separator = XMLHandler.getTagValue( stepnode, "separator" ); + enclosure = XMLHandler.getTagValue( stepnode, "enclosure" ); + breakInEnclosureAllowed = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "enclosure_breaks" ) ); + escapeCharacter = XMLHandler.getTagValue( stepnode, "escapechar" ); + header = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) ); + nrHeaderLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_headerlines" ), 1 ); + footer = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "footer" ) ); + nrFooterLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_footerlines" ), 1 ); + lineWrapped = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "line_wrapped" ) ); + nrWraps = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_wraps" ), 1 ); + layoutPaged = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "layout_paged" ) ); + nrLinesPerPage = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_per_page" ), 1 ); + nrLinesDocHeader = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_doc_header" ), 1 ); + String addToResult = XMLHandler.getTagValue( stepnode, "add_to_result_filenames" ); + if ( Const.isEmpty( addToResult ) ) { + isaddresult = true; + } else { + isaddresult = "Y".equalsIgnoreCase( addToResult ); + } + + String nempty = XMLHandler.getTagValue( stepnode, "noempty" ); + noEmptyLines = YES.equalsIgnoreCase( nempty ) || nempty == null; + includeFilename = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include" ) ); + filenameField = XMLHandler.getTagValue( stepnode, "include_field" ); + includeRowNumber = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownum" ) ); + rowNumberByFile = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownumByFile" ) ); + rowNumberField = XMLHandler.getTagValue( stepnode, "rownum_field" ); + fileFormat = XMLHandler.getTagValue( stepnode, "format" ); + encoding = XMLHandler.getTagValue( stepnode, "encoding" ); + + Node filenode = XMLHandler.getSubNode( stepnode, "file" ); + Node fields = XMLHandler.getSubNode( stepnode, "fields" ); + Node filtersNode = XMLHandler.getSubNode( stepnode, "filters" ); + int nrfiles = XMLHandler.countNodes( filenode, "name" ); + int nrfields = XMLHandler.countNodes( fields, "field" ); + int nrfilters = XMLHandler.countNodes( filtersNode, "filter" ); + + allocate( nrfiles, nrfields, nrfilters ); + + for ( int i = 0; i < nrfiles; i++ ) { + Node filenamenode = XMLHandler.getSubNodeByNr( filenode, "name", i ); + Node filemasknode = XMLHandler.getSubNodeByNr( filenode, "filemask", i ); + Node excludefilemasknode = XMLHandler.getSubNodeByNr( filenode, "exclude_filemask", i ); + Node fileRequirednode = XMLHandler.getSubNodeByNr( filenode, "file_required", i ); + Node includeSubFoldersnode = XMLHandler.getSubNodeByNr( filenode, "include_subfolders", i ); + fileName[i] = loadSource( filenode, filenamenode, i ); + fileMask[i] = XMLHandler.getNodeValue( filemasknode ); + excludeFileMask[i] = XMLHandler.getNodeValue( excludefilemasknode ); + fileRequired[i] = XMLHandler.getNodeValue( fileRequirednode ); + includeSubFolders[i] = XMLHandler.getNodeValue( includeSubFoldersnode ); + } + + fileType = XMLHandler.getTagValue( stepnode, "file", "type" ); + fileCompression = XMLHandler.getTagValue( stepnode, "file", "compression" ); + if ( fileCompression == null ) { + fileCompression = "None"; + if ( YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "zipped" ) ) ) { + fileCompression = "Zip"; + } + } + + // Backward compatibility : just one filter + if ( XMLHandler.getTagValue( stepnode, "filter" ) != null ) { + filter = new OldTextFileFilter[1]; + filter[0] = new OldTextFileFilter(); + + filter[0].setFilterPosition( Const.toInt( XMLHandler.getTagValue( stepnode, "filter_position" ), -1 ) ); + filter[0].setFilterString( XMLHandler.getTagValue( stepnode, "filter_string" ) ); + filter[0].setFilterLastLine( YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "filter_is_last_line" ) ) ); + filter[0].setFilterPositive( YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "filter_is_positive" ) ) ); + } else { + for ( int i = 0; i < nrfilters; i++ ) { + Node fnode = XMLHandler.getSubNodeByNr( filtersNode, "filter", i ); + filter[i] = new OldTextFileFilter(); + + filter[i].setFilterPosition( Const.toInt( XMLHandler.getTagValue( fnode, "filter_position" ), -1 ) ); + + String filterString = XMLHandler.getTagValue( fnode, "filter_string" ); + if ( filterString != null && filterString.startsWith( STRING_BASE64_PREFIX ) ) { + filter[i].setFilterString( new String( Base64.decodeBase64( filterString.substring( + STRING_BASE64_PREFIX.length() ).getBytes() ) ) ); + } else { + filter[i].setFilterString( filterString ); + } + + filter[i].setFilterLastLine( YES.equalsIgnoreCase( XMLHandler.getTagValue( fnode, "filter_is_last_line" ) ) ); + filter[i].setFilterPositive( YES.equalsIgnoreCase( XMLHandler.getTagValue( fnode, "filter_is_positive" ) ) ); + } + } + + for ( int i = 0; i < nrfields; i++ ) { + Node fnode = XMLHandler.getSubNodeByNr( fields, "field", i ); + TextFileInputField field = new TextFileInputField(); + + field.setName( XMLHandler.getTagValue( fnode, "name" ) ); + field.setType( ValueMeta.getType( XMLHandler.getTagValue( fnode, "type" ) ) ); + field.setFormat( XMLHandler.getTagValue( fnode, "format" ) ); + field.setCurrencySymbol( XMLHandler.getTagValue( fnode, "currency" ) ); + field.setDecimalSymbol( XMLHandler.getTagValue( fnode, "decimal" ) ); + field.setGroupSymbol( XMLHandler.getTagValue( fnode, "group" ) ); + field.setNullString( XMLHandler.getTagValue( fnode, "nullif" ) ); + field.setIfNullValue( XMLHandler.getTagValue( fnode, "ifnull" ) ); + field.setPosition( Const.toInt( XMLHandler.getTagValue( fnode, "position" ), -1 ) ); + field.setLength( Const.toInt( XMLHandler.getTagValue( fnode, "length" ), -1 ) ); + field.setPrecision( Const.toInt( XMLHandler.getTagValue( fnode, "precision" ), -1 ) ); + field.setTrimType( ValueMeta.getTrimTypeByCode( XMLHandler.getTagValue( fnode, "trim_type" ) ) ); + field.setRepeated( YES.equalsIgnoreCase( XMLHandler.getTagValue( fnode, "repeat" ) ) ); + + inputFields[i] = field; + } + + // Is there a limit on the number of rows we process? + rowLimit = Const.toLong( XMLHandler.getTagValue( stepnode, "limit" ), 0L ); + + errorIgnored = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_ignored" ) ); + skipBadFiles = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "skip_bad_files" ) ); + fileErrorField = XMLHandler.getTagValue( stepnode, "file_error_field" ); + fileErrorMessageField = XMLHandler.getTagValue( stepnode, "file_error_message_field" ); + errorLineSkipped = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_line_skipped" ) ); + errorCountField = XMLHandler.getTagValue( stepnode, "error_count_field" ); + errorFieldsField = XMLHandler.getTagValue( stepnode, "error_fields_field" ); + errorTextField = XMLHandler.getTagValue( stepnode, "error_text_field" ); + warningFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "bad_line_files_destination_directory" ); + warningFilesExtension = XMLHandler.getTagValue( stepnode, "bad_line_files_extension" ); + errorFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "error_line_files_destination_directory" ); + errorFilesExtension = XMLHandler.getTagValue( stepnode, "error_line_files_extension" ); + lineNumberFilesDestinationDirectory = + XMLHandler.getTagValue( stepnode, "line_number_files_destination_directory" ); + lineNumberFilesExtension = XMLHandler.getTagValue( stepnode, "line_number_files_extension" ); + // Backward compatible + + dateFormatLenient = !NO.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "date_format_lenient" ) ); + String dateLocale = XMLHandler.getTagValue( stepnode, "date_format_locale" ); + if ( dateLocale != null ) { + dateFormatLocale = EnvUtil.createLocale( dateLocale ); + } else { + dateFormatLocale = Locale.getDefault(); + } + + shortFileFieldName = XMLHandler.getTagValue( stepnode, "shortFileFieldName" ); + pathFieldName = XMLHandler.getTagValue( stepnode, "pathFieldName" ); + hiddenFieldName = XMLHandler.getTagValue( stepnode, "hiddenFieldName" ); + lastModificationTimeFieldName = XMLHandler.getTagValue( stepnode, "lastModificationTimeFieldName" ); + uriNameFieldName = XMLHandler.getTagValue( stepnode, "uriNameFieldName" ); + rootUriNameFieldName = XMLHandler.getTagValue( stepnode, "rootUriNameFieldName" ); + extensionFieldName = XMLHandler.getTagValue( stepnode, "extensionFieldName" ); + sizeFieldName = XMLHandler.getTagValue( stepnode, "sizeFieldName" ); + } catch ( Exception e ) { + throw new KettleXMLException( "Unable to load step info from XML", e ); + } + } + + public Object clone() { + OldTextFileInputMeta retval = (OldTextFileInputMeta) super.clone(); + + int nrfiles = fileName.length; + int nrfields = inputFields.length; + int nrfilters = filter.length; + + retval.allocate( nrfiles, nrfields, nrfilters ); + + for ( int i = 0; i < nrfiles; i++ ) { + retval.fileName[i] = fileName[i]; + retval.fileMask[i] = fileMask[i]; + retval.excludeFileMask[i] = excludeFileMask[i]; + retval.fileRequired[i] = fileRequired[i]; + retval.includeSubFolders[i] = includeSubFolders[i]; + } + + for ( int i = 0; i < nrfields; i++ ) { + retval.inputFields[i] = (TextFileInputField) inputFields[i].clone(); + } + + for ( int i = 0; i < nrfilters; i++ ) { + retval.filter[i] = (OldTextFileFilter) filter[i].clone(); + } + + retval.dateFormatLocale = (Locale) dateFormatLocale.clone(); + retval.fileCompression = fileCompression; + + return retval; + } + + public void allocate( int nrfiles, int nrfields, int nrfilters ) { + allocateFiles( nrfiles ); + + inputFields = new TextFileInputField[nrfields]; + filter = new OldTextFileFilter[nrfilters]; + } + + public void allocateFiles( int nrFiles ) { + fileName = new String[nrFiles]; + fileMask = new String[nrFiles]; + excludeFileMask = new String[nrFiles]; + fileRequired = new String[nrFiles]; + includeSubFolders = new String[nrFiles]; + } + + public void setDefault() { + shortFileFieldName = null; + pathFieldName = null; + hiddenFieldName = null; + lastModificationTimeFieldName = null; + uriNameFieldName = null; + rootUriNameFieldName = null; + extensionFieldName = null; + sizeFieldName = null; + + isaddresult = true; + separator = ";"; + enclosure = "\""; + breakInEnclosureAllowed = false; + header = true; + nrHeaderLines = 1; + footer = false; + nrFooterLines = 1; + lineWrapped = false; + nrWraps = 1; + layoutPaged = false; + nrLinesPerPage = 80; + nrLinesDocHeader = 0; + fileCompression = "None"; + noEmptyLines = true; + fileFormat = "DOS"; + fileType = "CSV"; + includeFilename = false; + filenameField = ""; + includeRowNumber = false; + rowNumberField = ""; + errorIgnored = false; + skipBadFiles = false; + errorLineSkipped = false; + warningFilesDestinationDirectory = null; + warningFilesExtension = "warning"; + errorFilesDestinationDirectory = null; + errorFilesExtension = "error"; + lineNumberFilesDestinationDirectory = null; + lineNumberFilesExtension = "line"; + dateFormatLenient = true; + rowNumberByFile = false; + + int nrfiles = 0; + int nrfields = 0; + int nrfilters = 0; + + allocate( nrfiles, nrfields, nrfilters ); + + for ( int i = 0; i < nrfiles; i++ ) { + fileName[i] = "filename" + ( i + 1 ); + fileMask[i] = ""; + excludeFileMask[i] = ""; + fileRequired[i] = NO; + includeSubFolders[i] = NO; + } + + for ( int i = 0; i < nrfields; i++ ) { + inputFields[i] = new TextFileInputField( "field" + ( i + 1 ), 1, -1 ); + } + + dateFormatLocale = Locale.getDefault(); + + rowLimit = 0L; + } + + public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep, + VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { + if ( !isPassingThruFields() ) { + // all incoming fields are not transmitted ! + row.clear(); + } else { + if ( info != null ) { + boolean found = false; + for ( int i = 0; i < info.length && !found; i++ ) { + if ( info[i] != null ) { + row.mergeRowMeta( info[i] ); + found = true; + } + } + } + } + + for ( int i = 0; i < inputFields.length; i++ ) { + TextFileInputField field = inputFields[i]; + + int type = field.getType(); + if ( type == ValueMetaInterface.TYPE_NONE ) { + type = ValueMetaInterface.TYPE_STRING; + } + + try { + ValueMetaInterface v = ValueMetaFactory.createValueMeta( field.getName(), type ); + v.setLength( field.getLength() ); + v.setPrecision( field.getPrecision() ); + v.setOrigin( name ); + v.setConversionMask( field.getFormat() ); + v.setDecimalSymbol( field.getDecimalSymbol() ); + v.setGroupingSymbol( field.getGroupSymbol() ); + v.setCurrencySymbol( field.getCurrencySymbol() ); + v.setDateFormatLenient( dateFormatLenient ); + v.setDateFormatLocale( dateFormatLocale ); + v.setTrimType( field.getTrimType() ); + + row.addValueMeta( v ); + } catch ( Exception e ) { + throw new KettleStepException( e ); + } + } + if ( errorIgnored ) { + if ( errorCountField != null && errorCountField.length() > 0 ) { + ValueMetaInterface v = new ValueMeta( errorCountField, ValueMetaInterface.TYPE_INTEGER ); + v.setLength( ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( errorFieldsField != null && errorFieldsField.length() > 0 ) { + ValueMetaInterface v = new ValueMeta( errorFieldsField, ValueMetaInterface.TYPE_STRING ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( errorTextField != null && errorTextField.length() > 0 ) { + ValueMetaInterface v = new ValueMeta( errorTextField, ValueMetaInterface.TYPE_STRING ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + } + if ( includeFilename ) { + ValueMetaInterface v = new ValueMeta( filenameField, ValueMetaInterface.TYPE_STRING ); + v.setLength( 100 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( includeRowNumber ) { + ValueMetaInterface v = new ValueMeta( rowNumberField, ValueMetaInterface.TYPE_INTEGER ); + v.setLength( ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + + // Add additional fields + + if ( getShortFileNameField() != null && getShortFileNameField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( getShortFileNameField() ), ValueMetaInterface.TYPE_STRING ); + v.setLength( 100, -1 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( getExtensionField() != null && getExtensionField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( getExtensionField() ), ValueMetaInterface.TYPE_STRING ); + v.setLength( 100, -1 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( getPathField() != null && getPathField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( getPathField() ), ValueMetaInterface.TYPE_STRING ); + v.setLength( 100, -1 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( getSizeField() != null && getSizeField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( getSizeField() ), ValueMetaInterface.TYPE_INTEGER ); + v.setOrigin( name ); + v.setLength( 9 ); + row.addValueMeta( v ); + } + if ( isHiddenField() != null && isHiddenField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( isHiddenField() ), ValueMetaInterface.TYPE_BOOLEAN ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + + if ( getLastModificationDateField() != null && getLastModificationDateField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( + space.environmentSubstitute( getLastModificationDateField() ), ValueMetaInterface.TYPE_DATE ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + if ( getUriField() != null && getUriField().length() > 0 ) { + ValueMetaInterface v = + new ValueMeta( space.environmentSubstitute( getUriField() ), ValueMetaInterface.TYPE_STRING ); + v.setLength( 100, -1 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + + if ( getRootUriField() != null && getRootUriField().length() > 0 ) { + ValueMetaInterface v = new ValueMeta( getRootUriField(), ValueMetaInterface.TYPE_STRING ); + v.setLength( 100, -1 ); + v.setOrigin( name ); + row.addValueMeta( v ); + } + + } + + @Override + @Deprecated + public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, + VariableSpace space ) throws KettleStepException { + getFields( inputRowMeta, name, info, nextStep, space, null, null ); + } + + public String getXML() { + StringBuffer retval = new StringBuffer( 1500 ); + + retval.append( " " ).append( XMLHandler.addTagValue( "accept_filenames", acceptingFilenames ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "passing_through_fields", passingThruFields ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "accept_field", acceptingField ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "accept_stepname", ( acceptingStep != null ? acceptingStep.getName() : "" ) ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "separator", separator ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "enclosure", enclosure ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "enclosure_breaks", breakInEnclosureAllowed ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "escapechar", escapeCharacter ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "header", header ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_headerlines", nrHeaderLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "footer", footer ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_footerlines", nrFooterLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "line_wrapped", lineWrapped ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_wraps", nrWraps ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "layout_paged", layoutPaged ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_per_page", nrLinesPerPage ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_doc_header", nrLinesDocHeader ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "noempty", noEmptyLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include", includeFilename ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include_field", filenameField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownum", includeRowNumber ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownumByFile", rowNumberByFile ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownum_field", rowNumberField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "format", fileFormat ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "encoding", encoding ) ); + retval.append( " " + XMLHandler.addTagValue( "add_to_result_filenames", isaddresult ) ); + + retval.append( " " ).append( Const.CR ); + for ( int i = 0; i < fileName.length; i++ ) { + saveSource( retval, fileName[i] ); + retval.append( " " ).append( XMLHandler.addTagValue( "filemask", fileMask[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "exclude_filemask", excludeFileMask[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_required", fileRequired[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", includeSubFolders[i] ) ); + } + retval.append( " " ).append( XMLHandler.addTagValue( "type", fileType ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "compression", + ( fileCompression == null ) ? "None" : fileCompression ) ); + retval.append( " " ).append( Const.CR ); + + retval.append( " " ).append( Const.CR ); + for ( int i = 0; i < filter.length; i++ ) { + String filterString = filter[i].getFilterString(); + byte[] filterBytes = new byte[] {}; + String filterPrefix = ""; + if ( filterString != null ) { + filterBytes = filterString.getBytes(); + filterPrefix = STRING_BASE64_PREFIX; + } + String filterEncoded = filterPrefix + new String( Base64.encodeBase64( filterBytes ) ); + + retval.append( " " ).append( Const.CR ); + retval.append( " " ).append( XMLHandler.addTagValue( "filter_string", filterEncoded, false ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "filter_position", filter[i].getFilterPosition(), false ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "filter_is_last_line", filter[i].isFilterLastLine(), false ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "filter_is_positive", filter[i].isFilterPositive(), false ) ); + retval.append( " " ).append( Const.CR ); + } + retval.append( " " ).append( Const.CR ); + + retval.append( " " ).append( Const.CR ); + for ( int i = 0; i < inputFields.length; i++ ) { + TextFileInputField field = inputFields[i]; + + retval.append( " " ).append( Const.CR ); + retval.append( " " ).append( XMLHandler.addTagValue( "name", field.getName() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "type", field.getTypeDesc() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "format", field.getFormat() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "currency", field.getCurrencySymbol() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "decimal", field.getDecimalSymbol() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "group", field.getGroupSymbol() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nullif", field.getNullString() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "ifnull", field.getIfNullValue() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "position", field.getPosition() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "length", field.getLength() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "precision", field.getPrecision() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "trim_type", field.getTrimTypeCode() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "repeat", field.isRepeated() ) ); + retval.append( " " ).append( Const.CR ); + } + retval.append( " " ).append( Const.CR ); + retval.append( " " ).append( XMLHandler.addTagValue( "limit", rowLimit ) ); + + // ERROR HANDLING + retval.append( " " ).append( XMLHandler.addTagValue( "error_ignored", errorIgnored ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "skip_bad_files", skipBadFiles ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_error_field", fileErrorField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_error_message_field", fileErrorMessageField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_line_skipped", errorLineSkipped ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_count_field", errorCountField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_fields_field", errorFieldsField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_text_field", errorTextField ) ); + + retval.append( " " ).append( + XMLHandler.addTagValue( "bad_line_files_destination_directory", warningFilesDestinationDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "bad_line_files_extension", warningFilesExtension ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "error_line_files_destination_directory", errorFilesDestinationDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_line_files_extension", errorFilesExtension ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "line_number_files_destination_directory", lineNumberFilesDestinationDirectory ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "line_number_files_extension", lineNumberFilesExtension ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "date_format_lenient", dateFormatLenient ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "date_format_locale", dateFormatLocale.toString() ) ); + + retval.append( " " ).append( XMLHandler.addTagValue( "shortFileFieldName", shortFileFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "pathFieldName", pathFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "hiddenFieldName", hiddenFieldName ) ); + retval.append( " " ).append( + XMLHandler.addTagValue( "lastModificationTimeFieldName", lastModificationTimeFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "uriNameFieldName", uriNameFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rootUriNameFieldName", rootUriNameFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "extensionFieldName", extensionFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "sizeFieldName", sizeFieldName ) ); + + return retval.toString(); + } + + public String getLookupStepname() { + if ( acceptingFilenames && acceptingStep != null && !Const.isEmpty( acceptingStep.getName() ) ) { + return acceptingStep.getName(); + } + return null; + } + + /** + * @param steps + * optionally search the info step in a list of steps + */ + public void searchInfoAndTargetSteps( List steps ) { + acceptingStep = StepMeta.findStep( steps, acceptingStepName ); + } + + public String[] getInfoSteps() { + if ( acceptingFilenames && acceptingStep != null ) { + return new String[] { acceptingStep.getName() }; + } + return null; + } + + public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List databases ) throws KettleException { + try { + acceptingFilenames = rep.getStepAttributeBoolean( id_step, "accept_filenames" ); + passingThruFields = rep.getStepAttributeBoolean( id_step, "passing_through_fields" ); + acceptingField = rep.getStepAttributeString( id_step, "accept_field" ); + acceptingStepName = rep.getStepAttributeString( id_step, "accept_stepname" ); + + separator = rep.getStepAttributeString( id_step, "separator" ); + enclosure = rep.getStepAttributeString( id_step, "enclosure" ); + breakInEnclosureAllowed = rep.getStepAttributeBoolean( id_step, "enclosure_breaks" ); + escapeCharacter = rep.getStepAttributeString( id_step, "escapechar" ); + header = rep.getStepAttributeBoolean( id_step, "header" ); + nrHeaderLines = (int) rep.getStepAttributeInteger( id_step, "nr_headerlines" ); + footer = rep.getStepAttributeBoolean( id_step, "footer" ); + nrFooterLines = (int) rep.getStepAttributeInteger( id_step, "nr_footerlines" ); + lineWrapped = rep.getStepAttributeBoolean( id_step, "line_wrapped" ); + nrWraps = (int) rep.getStepAttributeInteger( id_step, "nr_wraps" ); + layoutPaged = rep.getStepAttributeBoolean( id_step, "layout_paged" ); + nrLinesPerPage = (int) rep.getStepAttributeInteger( id_step, "nr_lines_per_page" ); + nrLinesDocHeader = (int) rep.getStepAttributeInteger( id_step, "nr_lines_doc_header" ); + noEmptyLines = rep.getStepAttributeBoolean( id_step, "noempty" ); + + includeFilename = rep.getStepAttributeBoolean( id_step, "include" ); + filenameField = rep.getStepAttributeString( id_step, "include_field" ); + includeRowNumber = rep.getStepAttributeBoolean( id_step, "rownum" ); + rowNumberByFile = rep.getStepAttributeBoolean( id_step, "rownumByFile" ); + rowNumberField = rep.getStepAttributeString( id_step, "rownum_field" ); + + fileFormat = rep.getStepAttributeString( id_step, "format" ); + encoding = rep.getStepAttributeString( id_step, "encoding" ); + String addToResult = rep.getStepAttributeString( id_step, "add_to_result_filenames" ); + if ( Const.isEmpty( addToResult ) ) { + isaddresult = true; + } else { + isaddresult = rep.getStepAttributeBoolean( id_step, "add_to_result_filenames" ); + } + + rowLimit = rep.getStepAttributeInteger( id_step, "limit" ); + + int nrfiles = rep.countNrStepAttributes( id_step, "file_name" ); + int nrfields = rep.countNrStepAttributes( id_step, "field_name" ); + int nrfilters = rep.countNrStepAttributes( id_step, "filter_string" ); + + allocate( nrfiles, nrfields, nrfilters ); + + for ( int i = 0; i < nrfiles; i++ ) { + fileName[i] = loadSourceRep( rep, id_step, i ); + fileMask[i] = rep.getStepAttributeString( id_step, i, "file_mask" ); + excludeFileMask[i] = rep.getStepAttributeString( id_step, i, "exclude_file_mask" ); + fileRequired[i] = rep.getStepAttributeString( id_step, i, "file_required" ); + if ( !YES.equalsIgnoreCase( fileRequired[i] ) ) { + fileRequired[i] = NO; + } + includeSubFolders[i] = rep.getStepAttributeString( id_step, i, "include_subfolders" ); + if ( !YES.equalsIgnoreCase( includeSubFolders[i] ) ) { + includeSubFolders[i] = NO; + } + } + fileType = rep.getStepAttributeString( id_step, "file_type" ); + fileCompression = rep.getStepAttributeString( id_step, "compression" ); + if ( fileCompression == null ) { + fileCompression = "None"; + if ( rep.getStepAttributeBoolean( id_step, "file_zipped" ) ) { + fileCompression = "Zip"; + } + } + + for ( int i = 0; i < nrfilters; i++ ) { + filter[i] = new OldTextFileFilter(); + filter[i].setFilterPosition( (int) rep.getStepAttributeInteger( id_step, i, "filter_position" ) ); + filter[i].setFilterString( rep.getStepAttributeString( id_step, i, "filter_string" ) ); + filter[i].setFilterLastLine( rep.getStepAttributeBoolean( id_step, i, "filter_is_last_line" ) ); + filter[i].setFilterPositive( rep.getStepAttributeBoolean( id_step, i, "filter_is_positive" ) ); + } + + for ( int i = 0; i < nrfields; i++ ) { + TextFileInputField field = new TextFileInputField(); + + field.setName( rep.getStepAttributeString( id_step, i, "field_name" ) ); + field.setType( ValueMeta.getType( rep.getStepAttributeString( id_step, i, "field_type" ) ) ); + field.setFormat( rep.getStepAttributeString( id_step, i, "field_format" ) ); + field.setCurrencySymbol( rep.getStepAttributeString( id_step, i, "field_currency" ) ); + field.setDecimalSymbol( rep.getStepAttributeString( id_step, i, "field_decimal" ) ); + field.setGroupSymbol( rep.getStepAttributeString( id_step, i, "field_group" ) ); + field.setNullString( rep.getStepAttributeString( id_step, i, "field_nullif" ) ); + field.setIfNullValue( rep.getStepAttributeString( id_step, i, "field_ifnull" ) ); + field.setPosition( (int) rep.getStepAttributeInteger( id_step, i, "field_position" ) ); + field.setLength( (int) rep.getStepAttributeInteger( id_step, i, "field_length" ) ); + field.setPrecision( (int) rep.getStepAttributeInteger( id_step, i, "field_precision" ) ); + field.setTrimType( ValueMeta + .getTrimTypeByCode( rep.getStepAttributeString( id_step, i, "field_trim_type" ) ) ); + field.setRepeated( rep.getStepAttributeBoolean( id_step, i, "field_repeat" ) ); + + inputFields[i] = field; + } + + errorIgnored = rep.getStepAttributeBoolean( id_step, "error_ignored" ); + skipBadFiles = rep.getStepAttributeBoolean( id_step, "skip_bad_files" ); + fileErrorField = rep.getStepAttributeString( id_step, "file_error_field" ); + fileErrorMessageField = rep.getStepAttributeString( id_step, "file_error_message_field" ); + + errorLineSkipped = rep.getStepAttributeBoolean( id_step, "error_line_skipped" ); + errorCountField = rep.getStepAttributeString( id_step, "error_count_field" ); + errorFieldsField = rep.getStepAttributeString( id_step, "error_fields_field" ); + errorTextField = rep.getStepAttributeString( id_step, "error_text_field" ); + + warningFilesDestinationDirectory = rep.getStepAttributeString( id_step, "bad_line_files_dest_dir" ); + warningFilesExtension = rep.getStepAttributeString( id_step, "bad_line_files_ext" ); + errorFilesDestinationDirectory = rep.getStepAttributeString( id_step, "error_line_files_dest_dir" ); + errorFilesExtension = rep.getStepAttributeString( id_step, "error_line_files_ext" ); + lineNumberFilesDestinationDirectory = rep.getStepAttributeString( id_step, "line_number_files_dest_dir" ); + lineNumberFilesExtension = rep.getStepAttributeString( id_step, "line_number_files_ext" ); + + dateFormatLenient = rep.getStepAttributeBoolean( id_step, 0, "date_format_lenient", true ); + + String dateLocale = rep.getStepAttributeString( id_step, 0, "date_format_locale" ); + if ( dateLocale != null ) { + dateFormatLocale = EnvUtil.createLocale( dateLocale ); + } else { + dateFormatLocale = Locale.getDefault(); + } + shortFileFieldName = rep.getStepAttributeString( id_step, "shortFileFieldName" ); + pathFieldName = rep.getStepAttributeString( id_step, "pathFieldName" ); + hiddenFieldName = rep.getStepAttributeString( id_step, "hiddenFieldName" ); + lastModificationTimeFieldName = rep.getStepAttributeString( id_step, "lastModificationTimeFieldName" ); + uriNameFieldName = rep.getStepAttributeString( id_step, "uriNameFieldName" ); + rootUriNameFieldName = rep.getStepAttributeString( id_step, "rootUriNameFieldName" ); + extensionFieldName = rep.getStepAttributeString( id_step, "extensionFieldName" ); + sizeFieldName = rep.getStepAttributeString( id_step, "sizeFieldName" ); + } catch ( Exception e ) { + throw new KettleException( "Unexpected error reading step information from the repository", e ); + } + } + + public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step ) throws KettleException { + try { + rep.saveStepAttribute( id_transformation, id_step, "accept_filenames", acceptingFilenames ); + rep.saveStepAttribute( id_transformation, id_step, "passing_through_fields", passingThruFields ); + rep.saveStepAttribute( id_transformation, id_step, "accept_field", acceptingField ); + rep.saveStepAttribute( id_transformation, id_step, "accept_stepname", ( acceptingStep != null + ? acceptingStep.getName() : "" ) ); + + rep.saveStepAttribute( id_transformation, id_step, "separator", separator ); + rep.saveStepAttribute( id_transformation, id_step, "enclosure", enclosure ); + rep.saveStepAttribute( id_transformation, id_step, "enclosure_breaks", breakInEnclosureAllowed ); + rep.saveStepAttribute( id_transformation, id_step, "escapechar", escapeCharacter ); + rep.saveStepAttribute( id_transformation, id_step, "header", header ); + rep.saveStepAttribute( id_transformation, id_step, "nr_headerlines", nrHeaderLines ); + rep.saveStepAttribute( id_transformation, id_step, "footer", footer ); + rep.saveStepAttribute( id_transformation, id_step, "nr_footerlines", nrFooterLines ); + rep.saveStepAttribute( id_transformation, id_step, "line_wrapped", lineWrapped ); + rep.saveStepAttribute( id_transformation, id_step, "nr_wraps", nrWraps ); + rep.saveStepAttribute( id_transformation, id_step, "layout_paged", layoutPaged ); + rep.saveStepAttribute( id_transformation, id_step, "nr_lines_per_page", nrLinesPerPage ); + rep.saveStepAttribute( id_transformation, id_step, "nr_lines_doc_header", nrLinesDocHeader ); + + rep.saveStepAttribute( id_transformation, id_step, "noempty", noEmptyLines ); + + rep.saveStepAttribute( id_transformation, id_step, "include", includeFilename ); + rep.saveStepAttribute( id_transformation, id_step, "include_field", filenameField ); + rep.saveStepAttribute( id_transformation, id_step, "rownum", includeRowNumber ); + rep.saveStepAttribute( id_transformation, id_step, "rownumByFile", rowNumberByFile ); + rep.saveStepAttribute( id_transformation, id_step, "rownum_field", rowNumberField ); + + rep.saveStepAttribute( id_transformation, id_step, "format", fileFormat ); + rep.saveStepAttribute( id_transformation, id_step, "encoding", encoding ); + rep.saveStepAttribute( id_transformation, id_step, "add_to_result_filenames", isaddresult ); + + rep.saveStepAttribute( id_transformation, id_step, "limit", rowLimit ); + + for ( int i = 0; i < fileName.length; i++ ) { + saveSourceRep( rep, id_transformation, id_step, i, fileName[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "file_mask", fileMask[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "exclude_file_mask", excludeFileMask[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "file_required", fileRequired[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "include_subfolders", includeSubFolders[i] ); + } + rep.saveStepAttribute( id_transformation, id_step, "file_type", fileType ); + rep.saveStepAttribute( id_transformation, id_step, "compression", + ( fileCompression == null ) ? "None" : fileCompression ); + + for ( int i = 0; i < filter.length; i++ ) { + rep.saveStepAttribute( id_transformation, id_step, i, "filter_position", filter[i].getFilterPosition() ); + rep.saveStepAttribute( id_transformation, id_step, i, "filter_string", filter[i].getFilterString() ); + rep.saveStepAttribute( id_transformation, id_step, i, "filter_is_last_line", filter[i].isFilterLastLine() ); + rep.saveStepAttribute( id_transformation, id_step, i, "filter_is_positive", filter[i].isFilterPositive() ); + } + + for ( int i = 0; i < inputFields.length; i++ ) { + TextFileInputField field = inputFields[i]; + + rep.saveStepAttribute( id_transformation, id_step, i, "field_name", field.getName() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_type", field.getTypeDesc() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_format", field.getFormat() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_currency", field.getCurrencySymbol() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_decimal", field.getDecimalSymbol() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_group", field.getGroupSymbol() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_nullif", field.getNullString() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_ifnull", field.getIfNullValue() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_position", field.getPosition() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_length", field.getLength() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_precision", field.getPrecision() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_trim_type", field.getTrimTypeCode() ); + rep.saveStepAttribute( id_transformation, id_step, i, "field_repeat", field.isRepeated() ); + } + + rep.saveStepAttribute( id_transformation, id_step, "error_ignored", errorIgnored ); + rep.saveStepAttribute( id_transformation, id_step, "skip_bad_files", skipBadFiles ); + rep.saveStepAttribute( id_transformation, id_step, "file_error_field", fileErrorField ); + rep.saveStepAttribute( id_transformation, id_step, "file_error_message_field", fileErrorMessageField ); + rep.saveStepAttribute( id_transformation, id_step, "error_line_skipped", errorLineSkipped ); + rep.saveStepAttribute( id_transformation, id_step, "error_count_field", errorCountField ); + rep.saveStepAttribute( id_transformation, id_step, "error_fields_field", errorFieldsField ); + rep.saveStepAttribute( id_transformation, id_step, "error_text_field", errorTextField ); + + rep.saveStepAttribute( + id_transformation, id_step, "bad_line_files_dest_dir", warningFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "bad_line_files_ext", warningFilesExtension ); + rep.saveStepAttribute( + id_transformation, id_step, "error_line_files_dest_dir", errorFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "error_line_files_ext", errorFilesExtension ); + rep.saveStepAttribute( + id_transformation, id_step, "line_number_files_dest_dir", lineNumberFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "line_number_files_ext", lineNumberFilesExtension ); + + rep.saveStepAttribute( id_transformation, id_step, "date_format_lenient", dateFormatLenient ); + rep.saveStepAttribute( id_transformation, id_step, "date_format_locale", dateFormatLocale.toString() ); + + rep.saveStepAttribute( id_transformation, id_step, "shortFileFieldName", shortFileFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "pathFieldName", pathFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "hiddenFieldName", hiddenFieldName ); + rep.saveStepAttribute( + id_transformation, id_step, "lastModificationTimeFieldName", lastModificationTimeFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "uriNameFieldName", uriNameFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "rootUriNameFieldName", rootUriNameFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "extensionFieldName", extensionFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "sizeFieldName", sizeFieldName ); + } catch ( Exception e ) { + throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e ); + } + } + + public String[] getFilePaths( VariableSpace space ) { + return FileInputList.createFilePathList( + space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); + } + + public FileInputList getTextFileList( VariableSpace space ) { + return FileInputList.createFileList( + space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); + } + + private boolean[] includeSubFolderBoolean() { + int len = fileName.length; + boolean[] includeSubFolderBoolean = new boolean[len]; + for ( int i = 0; i < len; i++ ) { + includeSubFolderBoolean[i] = YES.equalsIgnoreCase( includeSubFolders[i] ); + } + return includeSubFolderBoolean; + } + + public void check( List remarks, TransMeta transMeta, StepMeta stepMeta, + RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, + Repository repository, IMetaStore metaStore ) { + CheckResult cr; + + // See if we get input... + if ( input.length > 0 ) { + if ( !isAcceptingFilenames() ) { + cr = + new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString( + PKG, "TextFileInputMeta.CheckResult.NoInputError" ), stepMeta ); + remarks.add( cr ); + } else { + cr = + new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString( + PKG, "TextFileInputMeta.CheckResult.AcceptFilenamesOk" ), stepMeta ); + remarks.add( cr ); + } + } else { + cr = + new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString( + PKG, "TextFileInputMeta.CheckResult.NoInputOk" ), stepMeta ); + remarks.add( cr ); + } + + FileInputList textFileList = getTextFileList( transMeta ); + if ( textFileList.nrOfFiles() == 0 ) { + if ( !isAcceptingFilenames() ) { + cr = + new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString( + PKG, "TextFileInputMeta.CheckResult.ExpectedFilesError" ), stepMeta ); + remarks.add( cr ); + } + } else { + cr = + new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString( + PKG, "TextFileInputMeta.CheckResult.ExpectedFilesOk", "" + textFileList.nrOfFiles() ), stepMeta ); + remarks.add( cr ); + } + } + + public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, + TransMeta transMeta, Trans trans ) { + return new OldTextFileInput( stepMeta, stepDataInterface, cnr, transMeta, trans ); + } + + public StepDataInterface getStepData() { + return new OldTextFileInputData(); + } + + /** + * @return Returns the escapeCharacter. + */ + public String getEscapeCharacter() { + return escapeCharacter; + } + + /** + * @param escapeCharacter + * The escapeCharacter to set. + */ + public void setEscapeCharacter( String escapeCharacter ) { + this.escapeCharacter = escapeCharacter; + } + + public String getErrorCountField() { + return errorCountField; + } + + public void setErrorCountField( String errorCountField ) { + this.errorCountField = errorCountField; + } + + public String getErrorFieldsField() { + return errorFieldsField; + } + + public void setErrorFieldsField( String errorFieldsField ) { + this.errorFieldsField = errorFieldsField; + } + + public boolean isErrorIgnored() { + return errorIgnored; + } + + public void setErrorIgnored( boolean errorIgnored ) { + this.errorIgnored = errorIgnored; + } + + public String getErrorTextField() { + return errorTextField; + } + + public void setErrorTextField( String errorTextField ) { + this.errorTextField = errorTextField; + } + + /** + * @return Returns the lineWrapped. + */ + public boolean isLineWrapped() { + return lineWrapped; + } + + /** + * @param lineWrapped + * The lineWrapped to set. + */ + public void setLineWrapped( boolean lineWrapped ) { + this.lineWrapped = lineWrapped; + } + + /** + * @return Returns the nrFooterLines. + */ + public int getNrFooterLines() { + return nrFooterLines; + } + + /** + * @param nrFooterLines + * The nrFooterLines to set. + */ + public void setNrFooterLines( int nrFooterLines ) { + this.nrFooterLines = nrFooterLines; + } + + public String getRequiredFilesDesc( String tt ) { + if ( tt == null ) { + return RequiredFilesDesc[0]; + } + if ( tt.equals( RequiredFilesCode[1] ) ) { + return RequiredFilesDesc[1]; + } else { + return RequiredFilesDesc[0]; + } + } + + /** + * @return Returns the nrHeaderLines. + */ + public int getNrHeaderLines() { + return nrHeaderLines; + } + + /** + * @param nrHeaderLines + * The nrHeaderLines to set. + */ + public void setNrHeaderLines( int nrHeaderLines ) { + this.nrHeaderLines = nrHeaderLines; + } + + /** + * @return Returns the nrWraps. + */ + public int getNrWraps() { + return nrWraps; + } + + /** + * @param nrWraps + * The nrWraps to set. + */ + public void setNrWraps( int nrWraps ) { + this.nrWraps = nrWraps; + } + + /** + * @return Returns the layoutPaged. + */ + public boolean isLayoutPaged() { + return layoutPaged; + } + + /** + * @param layoutPaged + * The layoutPaged to set. + */ + public void setLayoutPaged( boolean layoutPaged ) { + this.layoutPaged = layoutPaged; + } + + /** + * @return Returns the nrLinesPerPage. + */ + public int getNrLinesPerPage() { + return nrLinesPerPage; + } + + /** + * @param nrLinesPerPage + * The nrLinesPerPage to set. + */ + public void setNrLinesPerPage( int nrLinesPerPage ) { + this.nrLinesPerPage = nrLinesPerPage; + } + + /** + * @return Returns the nrLinesDocHeader. + */ + public int getNrLinesDocHeader() { + return nrLinesDocHeader; + } + + /** + * @param nrLinesDocHeader + * The nrLinesDocHeader to set. + */ + public void setNrLinesDocHeader( int nrLinesDocHeader ) { + this.nrLinesDocHeader = nrLinesDocHeader; + } + + public String getWarningFilesDestinationDirectory() { + return warningFilesDestinationDirectory; + } + + public void setWarningFilesDestinationDirectory( String warningFilesDestinationDirectory ) { + this.warningFilesDestinationDirectory = warningFilesDestinationDirectory; + } + + public String getWarningFilesExtension() { + return warningFilesExtension; + } + + public void setWarningFilesExtension( String warningFilesExtension ) { + this.warningFilesExtension = warningFilesExtension; + } + + public String getLineNumberFilesDestinationDirectory() { + return lineNumberFilesDestinationDirectory; + } + + public void setLineNumberFilesDestinationDirectory( String lineNumberFilesDestinationDirectory ) { + this.lineNumberFilesDestinationDirectory = lineNumberFilesDestinationDirectory; + } + + public String getLineNumberFilesExtension() { + return lineNumberFilesExtension; + } + + public void setLineNumberFilesExtension( String lineNumberFilesExtension ) { + this.lineNumberFilesExtension = lineNumberFilesExtension; + } + + public String getErrorFilesDestinationDirectory() { + return errorFilesDestinationDirectory; + } + + public void setErrorFilesDestinationDirectory( String errorFilesDestinationDirectory ) { + this.errorFilesDestinationDirectory = errorFilesDestinationDirectory; + } + + public String getErrorLineFilesExtension() { + return errorFilesExtension; + } + + public void setErrorLineFilesExtension( String errorLineFilesExtension ) { + this.errorFilesExtension = errorLineFilesExtension; + } + + public boolean isDateFormatLenient() { + return dateFormatLenient; + } + + public void setDateFormatLenient( boolean dateFormatLenient ) { + this.dateFormatLenient = dateFormatLenient; + } + + /** + * @param isaddresult + * The isaddresult to set. + */ + public void setAddResultFile( boolean isaddresult ) { + this.isaddresult = isaddresult; + } + + /** + * @return Returns isaddresult. + */ + public boolean isAddResultFile() { + return isaddresult; + } + + public boolean isErrorLineSkipped() { + return errorLineSkipped; + } + + public void setErrorLineSkipped( boolean errorLineSkipped ) { + this.errorLineSkipped = errorLineSkipped; + } + + /** + * @return Returns the dateFormatLocale. + */ + public Locale getDateFormatLocale() { + return dateFormatLocale; + } + + /** + * @param dateFormatLocale + * The dateFormatLocale to set. + */ + public void setDateFormatLocale( Locale dateFormatLocale ) { + this.dateFormatLocale = dateFormatLocale; + } + + public boolean isAcceptingFilenames() { + return acceptingFilenames; + } + + public void setAcceptingFilenames( boolean getFileFromJob ) { + this.acceptingFilenames = getFileFromJob; + } + + public boolean isPassingThruFields() { + return passingThruFields; + } + + public void setPassingThruFields( boolean passingThruFields ) { + this.passingThruFields = passingThruFields; + } + + /** + * @return Returns the fileNameField. + */ + public String getAcceptingField() { + return acceptingField; + } + + /** + * @param fileNameField + * The fileNameField to set. + */ + public void setAcceptingField( String fileNameField ) { + this.acceptingField = fileNameField; + } + + /** + * @return Returns the acceptingStep. + */ + public String getAcceptingStepName() { + return acceptingStepName; + } + + /** + * @param acceptingStep + * The acceptingStep to set. + */ + public void setAcceptingStepName( String acceptingStep ) { + this.acceptingStepName = acceptingStep; + } + + /** + * @return Returns the acceptingStep. + */ + public StepMeta getAcceptingStep() { + return acceptingStep; + } + + /** + * @param acceptingStep + * The acceptingStep to set. + */ + public void setAcceptingStep( StepMeta acceptingStep ) { + this.acceptingStep = acceptingStep; + } + + public int getFileFormatTypeNr() { + // calculate the file format type in advance so we can use a switch + if ( getFileFormat().equalsIgnoreCase( "DOS" ) ) { + return FILE_FORMAT_DOS; + } else if ( getFileFormat().equalsIgnoreCase( "unix" ) ) { + return OldTextFileInputMeta.FILE_FORMAT_UNIX; + } else { + return OldTextFileInputMeta.FILE_FORMAT_MIXED; + } + } + + public int getFileTypeNr() { + // calculate the file type in advance CSV or Fixed? + if ( getFileType().equalsIgnoreCase( "CSV" ) ) { + return OldTextFileInputMeta.FILE_TYPE_CSV; + } else { + return OldTextFileInputMeta.FILE_TYPE_FIXED; + } + } + + @Override + public List getResourceDependencies( TransMeta transMeta, StepMeta stepInfo ) { + List references = new ArrayList( 5 ); + ResourceReference reference = new ResourceReference( stepInfo ); + references.add( reference ); + + String[] textFiles = getFilePaths( transMeta ); + if ( textFiles != null ) { + for ( int i = 0; i < textFiles.length; i++ ) { + reference.getEntries().add( new ResourceEntry( textFiles[i], ResourceType.FILE ) ); + } + } + return references; + } + + /** + * Since the exported transformation that runs this will reside in a ZIP file, we can't reference files relatively. So + * what this does is turn the name of files into absolute paths OR it simply includes the resource in the ZIP file. + * For now, we'll simply turn it into an absolute path and pray that the file is on a shared drive or something like + * that. + * + * @param space + * the variable space to use + * @param definitions + * @param resourceNamingInterface + * @param repository + * The repository to optionally load other resources from (to be converted to XML) + * @param metaStore + * the metaStore in which non-kettle metadata could reside. + * + * @return the filename of the exported resource + */ + public String exportResources( VariableSpace space, Map definitions, + ResourceNamingInterface resourceNamingInterface, Repository repository, IMetaStore metaStore ) throws KettleException { + try { + // The object that we're modifying here is a copy of the original! + // So let's change the filename from relative to absolute by grabbing the file object... + // In case the name of the file comes from previous steps, forget about this! + // + if ( !acceptingFilenames ) { + + // Replace the filename ONLY (folder or filename) + // + for ( int i = 0; i < fileName.length; i++ ) { + FileObject fileObject = KettleVFS.getFileObject( space.environmentSubstitute( fileName[i] ), space ); + fileName[i] = resourceNamingInterface.nameResource( fileObject, space, Const.isEmpty( fileMask[i] ) ); + } + } + return null; + } catch ( Exception e ) { + throw new KettleException( e ); + } + } + + @Override + public boolean supportsErrorHandling() { + return isErrorIgnored() && isSkipBadFiles(); + } + + @Override + public StepMetaInjectionInterface getStepMetaInjectionInterface() { + return new OldTextFileInputMetaInjection( this ); + } + + + @VisibleForTesting + public void setFileNameForTest( String[] fileName ) { + allocateFiles( fileName.length ); + setFileName( fileName ); + } + + protected String loadSource( Node filenode, Node filenamenode, int i ) { + return XMLHandler.getNodeValue( filenamenode ); + } + + protected void saveSource( StringBuffer retVal, String source ) { + retVal.append( " " ).append( XMLHandler.addTagValue( "name", source ) ); + } + + protected String loadSourceRep( Repository rep, ObjectId id_step, int i ) throws KettleException { + return rep.getStepAttributeString( id_step, i, "file_name" ); + } + + protected void saveSourceRep( Repository rep, ObjectId id_transformation, ObjectId id_step, int i, String fileName ) + throws KettleException { + rep.saveStepAttribute( id_transformation, id_step, i, "file_name", fileName ); //this should be in subclass + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjection.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjection.java new file mode 100644 index 000000000000..e2626340dcc5 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjection.java @@ -0,0 +1,688 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.pentaho.di.core.Const; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.trans.step.StepInjectionMetaEntry; +import org.pentaho.di.trans.step.StepMetaInjectionEntryInterface; +import org.pentaho.di.trans.step.StepMetaInjectionInterface; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; + +import static org.pentaho.di.trans.step.StepInjectionUtil.getEntry; + +/** + * This takes care of the external metadata injection into the TextFileInputMeta class + * + * @author Matt + */ +public class OldTextFileInputMetaInjection implements StepMetaInjectionInterface { + + enum Entry implements StepMetaInjectionEntryInterface { + + FILE_TYPE( ValueMetaInterface.TYPE_STRING, "File type (CSV/Fixed)" ), SEPARATOR( + ValueMetaInterface.TYPE_STRING, "The field separator" ), ENCLOSURE( + ValueMetaInterface.TYPE_STRING, "The field enclosure" ), ESCAPE_CHAR( + ValueMetaInterface.TYPE_STRING, "The escape character" ), BREAK_IN_ENCLOSURE( + ValueMetaInterface.TYPE_STRING, "Is a break allowed in an enclosure? (Y/N)" ), HEADER_PRESENT( + ValueMetaInterface.TYPE_STRING, "Is there a header present? (Y/N)" ), NR_HEADER_LINES( + ValueMetaInterface.TYPE_STRING, "The number of header lines" ), HAS_FOOTER( + ValueMetaInterface.TYPE_STRING, "Is there a footer present? (Y/N)" ), NR_FOOTER_LINES( + ValueMetaInterface.TYPE_STRING, "The number of footer lines" ), HAS_WRAPPED_LINES( + ValueMetaInterface.TYPE_STRING, "Are the lines wrapped? (Y/N)" ), NR_WRAPS( + ValueMetaInterface.TYPE_STRING, "The number of times a line is wrapped" ), HAS_PAGED_LAYOUT( + ValueMetaInterface.TYPE_STRING, "Is the layout paged? (Y/N)" ), NR_DOC_HEADER_LINES( + ValueMetaInterface.TYPE_STRING, "The number of document header lines" ), NR_LINES_PER_PAGE( + ValueMetaInterface.TYPE_STRING, "The number of lines per page" ), COMPRESSION_TYPE( + ValueMetaInterface.TYPE_STRING, "The compression type used (None, Zip or GZip)" ), NO_EMPTY_LINES( + ValueMetaInterface.TYPE_STRING, "Skip empty lines? (Y/N)" ), INCLUDE_FILENAME( + ValueMetaInterface.TYPE_STRING, "Include filename in the output? (Y/N)" ), FILENAME_FIELD( + ValueMetaInterface.TYPE_STRING, "The name of the filename field in the output" ), INCLUDE_ROW_NUMBER( + ValueMetaInterface.TYPE_STRING, "Include a row number in the output? (Y/N)" ), ROW_NUMBER_BY_FILE( + ValueMetaInterface.TYPE_STRING, "Reset the row number for each file? (Y/N)" ), ROW_NUMBER_FIELD( + ValueMetaInterface.TYPE_STRING, "The name of the row number field in the output" ), FILE_FORMAT( + ValueMetaInterface.TYPE_STRING, "File format (DOS, UNIX, mixed)" ), ENCODING( + ValueMetaInterface.TYPE_STRING, + "Encoding type (for allowed values see: http://wiki.pentaho.com/display/EAI/Text+File+Input)" ), ROW_LIMIT( + ValueMetaInterface.TYPE_STRING, "The maximum number of lines to read." ), DATE_FORMAT_LENIENT( + ValueMetaInterface.TYPE_STRING, "Use a lenient date parsing algorithm? (Y/N)" ), DATE_FORMAT_LOCALE( + ValueMetaInterface.TYPE_STRING, "The date format locale" ), ACCEPT_FILE_NAMES( + ValueMetaInterface.TYPE_STRING, "Accept file names? (Y/N)" ), ACCEPT_FILE_STEP( + ValueMetaInterface.TYPE_STRING, "The source step for the file names" ), ACCEPT_FILE_FIELD( + ValueMetaInterface.TYPE_STRING, "The input field for the file names" ), PASS_THROUGH_FIELDS( + ValueMetaInterface.TYPE_STRING, "Pass through fields? (Y/N)" ), ADD_FILES_TO_RESULT( + ValueMetaInterface.TYPE_STRING, "Add file names to the result files? (Y/N)" ), FILE_SHORT_FILE_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The short file output fieldname" ), FILE_PATH_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The path output fieldname" ), FILE_HIDDEN_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The hidden output fieldname" ), FILE_LAST_MODIFICATION_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The last modification time output fieldname" ), FILE_URI_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The URI output fieldname" ), FILE_EXTENSION_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The extension output fieldname" ), FILE_SIZE_FIELDNAME( + ValueMetaInterface.TYPE_STRING, "The file size output fieldname" ), SKIP_BAD_FILES( + ValueMetaInterface.TYPE_STRING, "Skip bad files? (Y/N)" ), FILE_ERROR_FIELD( + ValueMetaInterface.TYPE_STRING, "The output field for the error files" ), FILE_ERROR_MESSAGE_FIELD( + ValueMetaInterface.TYPE_STRING, "The output field for the file error messages" ), IGNORE_ERRORS( + ValueMetaInterface.TYPE_STRING, "Ignore errors? (Y/N)" ), ERROR_COUNT_FIELD( + ValueMetaInterface.TYPE_STRING, "The output field for the number of errors" ), ERROR_FIELDS_FIELD( + ValueMetaInterface.TYPE_STRING, "The output field for the fields in error" ), ERROR_TEXT_FIELD( + ValueMetaInterface.TYPE_STRING, "The output field for the error text" ), WARNING_FILES_TARGET_DIR( + ValueMetaInterface.TYPE_STRING, "The target directory for the warning files" ), WARNING_FILES_EXTENTION( + ValueMetaInterface.TYPE_STRING, "The warning files' extension" ), ERROR_FILES_TARGET_DIR( + ValueMetaInterface.TYPE_STRING, "The target directory for the error files" ), ERROR_FILES_EXTENTION( + ValueMetaInterface.TYPE_STRING, "The error files' extension" ), LINE_NR_FILES_TARGET_DIR( + ValueMetaInterface.TYPE_STRING, "The target directory for the line number files" ), + LINE_NR_FILES_EXTENTION( ValueMetaInterface.TYPE_STRING, "The line number files' extension" ), + ERROR_LINES_SKIPPED( ValueMetaInterface.TYPE_STRING, "Skip error lines? (Y/N)" ), + + FILENAME_LINES( ValueMetaInterface.TYPE_NONE, "The list of file definitions" ), FILENAME_LINE( + ValueMetaInterface.TYPE_NONE, "One file definition line" ), FILENAME( + ValueMetaInterface.TYPE_STRING, "The filename or directory" ), FILEMASK( + ValueMetaInterface.TYPE_STRING, "The file mask (regex)" ), EXCLUDE_FILEMASK( + ValueMetaInterface.TYPE_STRING, "The mask for the files to exclude (regex)" ), FILE_REQUIRED( + ValueMetaInterface.TYPE_STRING, "Is this a required file (Y/N)" ), INCLUDE_SUBFOLDERS( + ValueMetaInterface.TYPE_STRING, "Include sub-folders when searching files? (Y/N)" ), + + FIELDS( ValueMetaInterface.TYPE_NONE, "The fields" ), FIELD( ValueMetaInterface.TYPE_NONE, "One field" ), + FIELD_NAME( ValueMetaInterface.TYPE_STRING, "Name" ), FIELD_POSITION( + ValueMetaInterface.TYPE_STRING, "Position" ), FIELD_LENGTH( ValueMetaInterface.TYPE_STRING, "Length" ), + FIELD_TYPE( ValueMetaInterface.TYPE_STRING, "Data type (String, Number, ...)" ), FIELD_IGNORE( + ValueMetaInterface.TYPE_STRING, "Ignore? (Y/N)" ), + FIELD_FORMAT( ValueMetaInterface.TYPE_STRING, "Format" ), FIELD_TRIM_TYPE( + ValueMetaInterface.TYPE_STRING, "Trim type (none, left, right, both)" ), FIELD_PRECISION( + ValueMetaInterface.TYPE_STRING, "Precision" ), FIELD_DECIMAL( + ValueMetaInterface.TYPE_STRING, "Decimal symbol" ), FIELD_GROUP( + ValueMetaInterface.TYPE_STRING, "Grouping symbol" ), FIELD_CURRENCY( + ValueMetaInterface.TYPE_STRING, "Currency symbol" ), FIELD_REPEAT( + ValueMetaInterface.TYPE_STRING, "Repeat values? (Y/N)" ), FIELD_NULL_STRING( + ValueMetaInterface.TYPE_STRING, "The null string" ), FIELD_IF_NULL( + ValueMetaInterface.TYPE_STRING, "The default value if null" ), + + // The filters + // + FILTERS( ValueMetaInterface.TYPE_NONE, "The filter definitions" ), FILTER( + ValueMetaInterface.TYPE_NONE, "One filter definition" ), FILTER_POSITION( + ValueMetaInterface.TYPE_STRING, "Position" ), FILTER_STRING( + ValueMetaInterface.TYPE_STRING, "Filter string" ), FILTER_LAST_LINE( + ValueMetaInterface.TYPE_STRING, "Stop reading when filter found? (Y/N)" ), FILTER_POSITIVE( + ValueMetaInterface.TYPE_STRING, "Only match the filter lines? (Y/N)" ); + + private int valueType; + private String description; + + private Entry( int valueType, String description ) { + this.valueType = valueType; + this.description = description; + } + + /** + * @return the valueType + */ + public int getValueType() { + return valueType; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + public static Entry findEntry( String key ) { + return Entry.valueOf( key ); + } + + public static Entry[] getTopEntries() { + return new Entry[] { + FILE_TYPE, SEPARATOR, ENCLOSURE, ESCAPE_CHAR, BREAK_IN_ENCLOSURE, + HEADER_PRESENT, NR_HEADER_LINES, HAS_FOOTER, NR_FOOTER_LINES, + HAS_WRAPPED_LINES, NR_WRAPS, HAS_PAGED_LAYOUT, NR_DOC_HEADER_LINES, + NR_LINES_PER_PAGE, COMPRESSION_TYPE, NO_EMPTY_LINES, INCLUDE_FILENAME, + FILENAME_FIELD, INCLUDE_ROW_NUMBER, ROW_NUMBER_BY_FILE, ROW_NUMBER_FIELD, + FILE_FORMAT, ENCODING, ROW_LIMIT, DATE_FORMAT_LENIENT, DATE_FORMAT_LOCALE, + ACCEPT_FILE_NAMES, ACCEPT_FILE_STEP, ACCEPT_FILE_FIELD, PASS_THROUGH_FIELDS, + ADD_FILES_TO_RESULT, FILE_SHORT_FILE_FIELDNAME, FILE_PATH_FIELDNAME, + FILE_HIDDEN_FIELDNAME, FILE_LAST_MODIFICATION_FIELDNAME, FILE_URI_FIELDNAME, + FILE_EXTENSION_FIELDNAME, FILE_SIZE_FIELDNAME, SKIP_BAD_FILES, + FILE_ERROR_FIELD, FILE_ERROR_MESSAGE_FIELD, IGNORE_ERRORS, ERROR_COUNT_FIELD, + ERROR_FIELDS_FIELD, ERROR_TEXT_FIELD, WARNING_FILES_TARGET_DIR, + WARNING_FILES_EXTENTION, ERROR_FILES_TARGET_DIR, ERROR_FILES_EXTENTION, + LINE_NR_FILES_TARGET_DIR, LINE_NR_FILES_EXTENTION, ERROR_LINES_SKIPPED + }; + } + + public static Entry[] getFileFieldsEntries() { + return new Entry[] { FILENAME, FILEMASK, EXCLUDE_FILEMASK, FILE_REQUIRED, INCLUDE_SUBFOLDERS }; + } + + public static Entry[] getAggEntries() { + return new Entry[] { + FIELD_NAME, FIELD_POSITION, FIELD_LENGTH, FIELD_TYPE, FIELD_IGNORE, + FIELD_FORMAT, FIELD_TRIM_TYPE, FIELD_PRECISION, FIELD_DECIMAL, + FIELD_GROUP, FIELD_CURRENCY, FIELD_REPEAT, FIELD_NULL_STRING, + FIELD_IF_NULL }; + } + } + + + private final OldTextFileInputMeta meta; + + public OldTextFileInputMetaInjection( OldTextFileInputMeta meta ) { + this.meta = meta; + } + + @Override + public List getStepInjectionMetadataEntries() throws KettleException { + List all = new ArrayList(); + + for ( Entry topEntry : Entry.getTopEntries() ) { + all.add( getEntry( topEntry ) ); + } + + // The file name lines + // + StepInjectionMetaEntry filesEntry = getEntry( Entry.FILENAME_LINES ); + all.add( filesEntry ); + StepInjectionMetaEntry fileEntry = getEntry( Entry.FILENAME_LINE ); + filesEntry.getDetails().add( fileEntry ); + + Entry[] fileFieldsEntries = Entry.getFileFieldsEntries(); + List fileEntryDetails = fileEntry.getDetails(); + for ( Entry entry : fileFieldsEntries ) { + StepInjectionMetaEntry metaEntry = getEntry( entry ); + fileEntryDetails.add( metaEntry ); + } + + // The fields... + // + StepInjectionMetaEntry fieldsEntry = getEntry( Entry.FIELDS ); + all.add( fieldsEntry ); + StepInjectionMetaEntry fieldEntry = getEntry( Entry.FIELD ); + fieldsEntry.getDetails().add( fieldEntry ); + + Entry[] aggEntries = Entry.getAggEntries(); + List fieldEntryDetails = fieldEntry.getDetails(); + for ( Entry entry : aggEntries ) { + StepInjectionMetaEntry metaEntry = getEntry( entry ); + fieldEntryDetails.add( metaEntry ); + } + + return all; + } + + private static class FileLine { + String filename; + String includeMask; + String excludeMask; + String required; + String includeSubfolders; + } + + @Override + public void injectStepMetadataEntries( List all ) throws KettleException { + + List fileLines = new ArrayList(); + List fields = new ArrayList(); + List filters = new ArrayList(); + + // Parse the fields, inject into the meta class.. + // + for ( StepInjectionMetaEntry lookFields : all ) { + Entry fieldsEntry = Entry.findEntry( lookFields.getKey() ); + if ( fieldsEntry == null ) { + continue; + } + + String lookValue = (String) lookFields.getValue(); + switch ( fieldsEntry ) { + case FILENAME_LINES: + for ( StepInjectionMetaEntry lookField : lookFields.getDetails() ) { + Entry fieldEntry = Entry.findEntry( lookField.getKey() ); + if ( fieldEntry == Entry.FILENAME_LINE ) { + FileLine fileLine = new FileLine(); + + List entries = lookField.getDetails(); + for ( StepInjectionMetaEntry entry : entries ) { + Entry metaEntry = Entry.findEntry( entry.getKey() ); + if ( metaEntry != null ) { + String value = (String) entry.getValue(); + switch ( metaEntry ) { + case FILENAME: + fileLine.filename = value; + break; + case FILEMASK: + fileLine.includeMask = value; + break; + case EXCLUDE_FILEMASK: + fileLine.excludeMask = value; + break; + case FILE_REQUIRED: + fileLine.required = value; + break; + case INCLUDE_SUBFOLDERS: + fileLine.includeSubfolders = value; + break; + default: + break; + } + } + } + fileLines.add( fileLine ); + } + } + break; + + case FIELDS: + for ( StepInjectionMetaEntry lookField : lookFields.getDetails() ) { + Entry fieldEntry = Entry.findEntry( lookField.getKey() ); + if ( fieldEntry == Entry.FIELD ) { + + TextFileInputField field = new TextFileInputField(); + + List entries = lookField.getDetails(); + for ( StepInjectionMetaEntry entry : entries ) { + Entry metaEntry = Entry.findEntry( entry.getKey() ); + if ( metaEntry != null ) { + String value = (String) entry.getValue(); + switch ( metaEntry ) { + case FIELD_NAME: + field.setName( value ); + break; + case FIELD_POSITION: + field.setPosition( Const.toInt( value, -1 ) ); + break; + case FIELD_LENGTH: + field.setLength( Const.toInt( value, -1 ) ); + break; + case FIELD_TYPE: + field.setType( ValueMeta.getType( value ) ); + break; + case FIELD_IGNORE: + field.setIgnored( "Y".equalsIgnoreCase( value ) ); + break; + case FIELD_FORMAT: + field.setFormat( value ); + break; + case FIELD_TRIM_TYPE: + field.setTrimType( ValueMeta.getTrimTypeByCode( value ) ); + break; + case FIELD_PRECISION: + field.setPrecision( Const.toInt( value, -1 ) ); + break; + case FIELD_DECIMAL: + field.setDecimalSymbol( value ); + break; + case FIELD_GROUP: + field.setGroupSymbol( value ); + break; + case FIELD_CURRENCY: + field.setCurrencySymbol( value ); + break; + case FIELD_REPEAT: + field.setRepeated( "Y".equalsIgnoreCase( value ) ); + break; + case FIELD_NULL_STRING: + field.setNullString( value ); + break; + case FIELD_IF_NULL: + field.setIfNullValue( value ); + break; + default: + break; + } + } + } + fields.add( field ); + } + } + break; + + case FILTERS: + for ( StepInjectionMetaEntry lookField : lookFields.getDetails() ) { + Entry fieldEntry = Entry.findEntry( lookField.getKey() ); + if ( fieldEntry == Entry.FILTER ) { + OldTextFileFilter filterLine = new OldTextFileFilter(); + + List entries = lookField.getDetails(); + for ( StepInjectionMetaEntry entry : entries ) { + Entry metaEntry = Entry.findEntry( entry.getKey() ); + if ( metaEntry != null ) { + String value = (String) entry.getValue(); + switch ( metaEntry ) { + case FILTER_POSITION: + filterLine.setFilterPosition( Const.toInt( value, 0 ) ); + break; + case FILTER_STRING: + filterLine.setFilterString( value ); + break; + case FILTER_LAST_LINE: + filterLine.setFilterLastLine( "Y".equalsIgnoreCase( value ) ); + break; + case FILTER_POSITIVE: + filterLine.setFilterPositive( "Y".equalsIgnoreCase( value ) ); + break; + default: + break; + } + } + } + filters.add( filterLine ); + } + } + break; + + case FILE_TYPE: + meta.setFileType( lookValue ); + break; + case SEPARATOR: + meta.setSeparator( lookValue ); + break; + case ENCLOSURE: + meta.setEnclosure( lookValue ); + break; + case ESCAPE_CHAR: + meta.setEscapeCharacter( lookValue ); + break; + case BREAK_IN_ENCLOSURE: + meta.setBreakInEnclosureAllowed( "Y".equalsIgnoreCase( lookValue ) ); + break; + case HEADER_PRESENT: + meta.setHeader( "Y".equalsIgnoreCase( lookValue ) ); + break; + case NR_HEADER_LINES: + meta.setNrHeaderLines( Const.toInt( lookValue, -1 ) ); + break; + case HAS_FOOTER: + meta.setFooter( "Y".equalsIgnoreCase( lookValue ) ); + break; + case NR_FOOTER_LINES: + meta.setNrFooterLines( Const.toInt( lookValue, -1 ) ); + break; + case HAS_WRAPPED_LINES: + meta.setLineWrapped( "Y".equalsIgnoreCase( lookValue ) ); + break; + case NR_WRAPS: + meta.setNrWraps( Const.toInt( lookValue, -1 ) ); + break; + case HAS_PAGED_LAYOUT: + meta.setLayoutPaged( "Y".equalsIgnoreCase( lookValue ) ); + break; + case NR_DOC_HEADER_LINES: + meta.setNrLinesDocHeader( Const.toInt( lookValue, -1 ) ); + break; + case NR_LINES_PER_PAGE: + meta.setNrLinesPerPage( Const.toInt( lookValue, -1 ) ); + break; + case COMPRESSION_TYPE: + meta.setFileCompression( lookValue ); + break; + case NO_EMPTY_LINES: + meta.setNoEmptyLines( "Y".equalsIgnoreCase( lookValue ) ); + break; + case INCLUDE_FILENAME: + meta.setIncludeFilename( "Y".equalsIgnoreCase( lookValue ) ); + break; + case FILENAME_FIELD: + meta.setFilenameField( lookValue ); + break; + case INCLUDE_ROW_NUMBER: + meta.setIncludeRowNumber( "Y".equalsIgnoreCase( lookValue ) ); + break; + case ROW_NUMBER_BY_FILE: + meta.setRowNumberByFile( "Y".equalsIgnoreCase( lookValue ) ); + break; + case ROW_NUMBER_FIELD: + meta.setRowNumberField( lookValue ); + break; + case FILE_FORMAT: + meta.setFileFormat( lookValue ); + break; + case ENCODING: + meta.setEncoding( lookValue ); + break; + case ROW_LIMIT: + meta.setRowLimit( Const.toInt( lookValue, -1 ) ); + break; + case DATE_FORMAT_LENIENT: + meta.setDateFormatLenient( "Y".equalsIgnoreCase( lookValue ) ); + break; + case DATE_FORMAT_LOCALE: + meta.setDateFormatLocale( new Locale( lookValue ) ); + break; + case ACCEPT_FILE_NAMES: + meta.setAcceptingFilenames( "Y".equalsIgnoreCase( lookValue ) ); + break; + case ACCEPT_FILE_STEP: + meta.setAcceptingStepName( lookValue ); + break; + case ACCEPT_FILE_FIELD: + meta.setAcceptingField( lookValue ); + break; + case PASS_THROUGH_FIELDS: + meta.setPassingThruFields( "Y".equalsIgnoreCase( lookValue ) ); + break; + case ADD_FILES_TO_RESULT: + meta.setAddResultFile( "Y".equalsIgnoreCase( lookValue ) ); + break; + case FILE_SHORT_FILE_FIELDNAME: + meta.setShortFileNameField( lookValue ); + break; + case FILE_PATH_FIELDNAME: + meta.setPathField( lookValue ); + break; + case FILE_HIDDEN_FIELDNAME: + meta.setIsHiddenField( lookValue ); + break; + case FILE_LAST_MODIFICATION_FIELDNAME: + meta.setLastModificationDateField( lookValue ); + break; + case FILE_URI_FIELDNAME: + meta.setUriField( lookValue ); + break; + case FILE_EXTENSION_FIELDNAME: + meta.setExtensionField( lookValue ); + break; + case FILE_SIZE_FIELDNAME: + meta.setSizeField( lookValue ); + break; + case SKIP_BAD_FILES: + meta.setSkipBadFiles( "Y".equalsIgnoreCase( lookValue ) ); + break; + case FILE_ERROR_FIELD: + meta.setFileErrorField( lookValue ); + break; + case FILE_ERROR_MESSAGE_FIELD: + meta.setFileErrorMessageField( lookValue ); + break; + case IGNORE_ERRORS: + meta.setErrorIgnored( "Y".equalsIgnoreCase( lookValue ) ); + break; + case ERROR_COUNT_FIELD: + meta.setErrorCountField( lookValue ); + break; + case ERROR_FIELDS_FIELD: + meta.setErrorFieldsField( lookValue ); + break; + case ERROR_TEXT_FIELD: + meta.setErrorTextField( lookValue ); + break; + case WARNING_FILES_TARGET_DIR: + meta.setWarningFilesDestinationDirectory( lookValue ); + break; + case WARNING_FILES_EXTENTION: + meta.setWarningFilesExtension( lookValue ); + break; + case ERROR_FILES_TARGET_DIR: + meta.setErrorFilesDestinationDirectory( lookValue ); + break; + case ERROR_FILES_EXTENTION: + meta.setErrorLineFilesExtension( lookValue ); + break; + case LINE_NR_FILES_TARGET_DIR: + meta.setLineNumberFilesDestinationDirectory( lookValue ); + break; + case LINE_NR_FILES_EXTENTION: + meta.setLineNumberFilesExtension( lookValue ); + break; + case ERROR_LINES_SKIPPED: + meta.setErrorLineSkipped( "Y".equalsIgnoreCase( lookValue ) ); + break; + default: + break; + } + } + + // Pass the grid to the step metadata + // Only change a list when you need to, don't clear/reset existing content if you don't send new content. + // + if ( fields.size() > 0 ) { + meta.setInputFields( fields.toArray( new TextFileInputField[fields.size()] ) ); + } + if ( fileLines.size() > 0 ) { + meta.allocateFiles( fileLines.size() ); + //CHECKSTYLE:Indentation:OFF + for ( int i = 0; i < fileLines.size(); i++ ) { + FileLine fileLine = fileLines.get( i ); + meta.getFileName()[i] = fileLine.filename; + meta.getFileMask()[i] = fileLine.includeMask; + meta.getExludeFileMask()[i] = fileLine.excludeMask; + meta.getExludeFileMask()[i] = fileLine.excludeMask; + meta.getFileRequired()[i] = fileLine.required; + meta.getIncludeSubFolders()[i] = fileLine.includeSubfolders; + } + } + if ( filters.size() > 0 ) { + meta.setFilter( filters.toArray( new OldTextFileFilter[filters.size()] ) ); + } + + } + + public List extractStepMetadataEntries() throws KettleException { + List result = new ArrayList(); + result.add( getEntry( Entry.FILE_TYPE, meta.getFileType() ) ); + result.add( getEntry( Entry.SEPARATOR, meta.getSeparator() ) ); + result.add( getEntry( Entry.ENCLOSURE, meta.getEnclosure() ) ); + result.add( getEntry( Entry.ESCAPE_CHAR, meta.getEscapeCharacter() ) ); + result.add( getEntry( Entry.BREAK_IN_ENCLOSURE, meta.isBreakInEnclosureAllowed() ) ); + result.add( getEntry( Entry.HEADER_PRESENT, meta.hasHeader() ) ); + result.add( getEntry( Entry.NR_HEADER_LINES, meta.getNrHeaderLines() ) ); + result.add( getEntry( Entry.HAS_FOOTER, meta.hasFooter() ) ); + result.add( getEntry( Entry.NR_FOOTER_LINES, meta.getNrFooterLines() ) ); + result.add( getEntry( Entry.HAS_WRAPPED_LINES, meta.isLineWrapped() ) ); + result.add( getEntry( Entry.NR_WRAPS, meta.getNrWraps() ) ); + result.add( getEntry( Entry.HAS_PAGED_LAYOUT, meta.isLayoutPaged() ) ); + result.add( getEntry( Entry.NR_DOC_HEADER_LINES, meta.getNrLinesDocHeader() ) ); + result.add( getEntry( Entry.NR_LINES_PER_PAGE, meta.getNrLinesPerPage() ) ); + result.add( getEntry( Entry.COMPRESSION_TYPE, meta.getFileCompression() ) ); + result.add( getEntry( Entry.NO_EMPTY_LINES, meta.noEmptyLines() ) ); + result.add( getEntry( Entry.INCLUDE_FILENAME, meta.includeFilename() ) ); + result.add( getEntry( Entry.FILENAME_FIELD, meta.getFilenameField() ) ); + result.add( getEntry( Entry.INCLUDE_ROW_NUMBER, meta.includeRowNumber() ) ); + result.add( getEntry( Entry.ROW_NUMBER_BY_FILE, meta.isRowNumberByFile() ) ); + result.add( getEntry( Entry.ROW_NUMBER_FIELD, meta.getRowNumberField() ) ); + result.add( getEntry( Entry.FILE_FORMAT, meta.getFileFormat() ) ); + result.add( getEntry( Entry.ENCODING, meta.getEncoding() ) ); + result.add( getEntry( Entry.ROW_LIMIT, meta.getRowLimit() ) ); + result.add( getEntry( Entry.DATE_FORMAT_LENIENT, meta.isDateFormatLenient() ) ); + result.add( getEntry( Entry.DATE_FORMAT_LOCALE, meta.getDateFormatLocale() ) ); + result.add( getEntry( Entry.ACCEPT_FILE_NAMES, meta.isAcceptingFilenames() ) ); + result.add( getEntry( Entry.ACCEPT_FILE_STEP, meta.getAcceptingStepName() ) ); + result.add( getEntry( Entry.ACCEPT_FILE_FIELD, meta.getAcceptingField() ) ); + result.add( getEntry( Entry.PASS_THROUGH_FIELDS, meta.isPassingThruFields() ) ); + result.add( getEntry( Entry.ADD_FILES_TO_RESULT, meta.isAddResultFile() ) ); + result.add( getEntry( Entry.FILE_SHORT_FILE_FIELDNAME, meta.getShortFileNameField() ) ); + result.add( getEntry( Entry.FILE_PATH_FIELDNAME, meta.getPathField() ) ); + result.add( getEntry( Entry.FILE_HIDDEN_FIELDNAME, meta.isHiddenField() ) ); + result.add( getEntry( Entry.FILE_LAST_MODIFICATION_FIELDNAME, meta.getLastModificationDateField() ) ); + result.add( getEntry( Entry.FILE_URI_FIELDNAME, meta.getUriField() ) ); + result.add( getEntry( Entry.FILE_EXTENSION_FIELDNAME, meta.getExtensionField() ) ); + result.add( getEntry( Entry.FILE_SIZE_FIELDNAME, meta.getSizeField() ) ); + result.add( getEntry( Entry.SKIP_BAD_FILES, meta.isSkipBadFiles() ) ); + result.add( getEntry( Entry.FILE_ERROR_FIELD, meta.getFileErrorField() ) ); + result.add( getEntry( Entry.FILE_ERROR_MESSAGE_FIELD, meta.getFileErrorMessageField() ) ); + result.add( getEntry( Entry.IGNORE_ERRORS, meta.isErrorIgnored() ) ); + result.add( getEntry( Entry.ERROR_COUNT_FIELD, meta.getErrorCountField() ) ); + result.add( getEntry( Entry.ERROR_FIELDS_FIELD, meta.getErrorFieldsField() ) ); + result.add( getEntry( Entry.ERROR_TEXT_FIELD, meta.getErrorTextField() ) ); + result.add( getEntry( Entry.WARNING_FILES_TARGET_DIR, meta.getWarningFilesDestinationDirectory() ) ); + result.add( getEntry( Entry.WARNING_FILES_EXTENTION, meta.getWarningFilesExtension() ) ); + result.add( getEntry( Entry.ERROR_FILES_TARGET_DIR, meta.getErrorFilesDestinationDirectory() ) ); + result.add( getEntry( Entry.ERROR_FILES_EXTENTION, meta.getErrorLineFilesExtension() ) ); + result.add( getEntry( Entry.LINE_NR_FILES_TARGET_DIR, meta.getLineNumberFilesDestinationDirectory() ) ); + result.add( getEntry( Entry.LINE_NR_FILES_EXTENTION, meta.getLineNumberFilesExtension() ) ); + result.add( getEntry( Entry.ERROR_LINES_SKIPPED, meta.isErrorLineSkipped() ) ); + + StepInjectionMetaEntry filenameLinesEntry = getEntry( Entry.FILENAME_LINES ); + if ( !Const.isEmpty( meta.getFileName() ) ) { + for ( int i = 0, len = meta.getFileName().length; i < len; i++ ) { + StepInjectionMetaEntry filenameLineEntry = getEntry( Entry.FILENAME_LINE ); + filenameLinesEntry.getDetails().add( filenameLineEntry ); + + List filenameLineEntryDetails = filenameLineEntry.getDetails(); + filenameLineEntryDetails.add( getEntry( Entry.FILENAME, meta.getFileName()[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.FILEMASK, meta.getFileMask()[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.EXCLUDE_FILEMASK, meta.getExludeFileMask()[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.FILE_REQUIRED, meta.getFileRequired()[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.INCLUDE_SUBFOLDERS, meta.getIncludeSubFolders()[ i ] ) ); + } + } + result.add( filenameLinesEntry ); + + StepInjectionMetaEntry fieldsEntry = getEntry( Entry.FIELDS ); + if ( !Const.isEmpty( meta.getInputFields() ) ) { + for ( TextFileInputField inputField : meta.getInputFields() ) { + StepInjectionMetaEntry fieldEntry = getEntry( Entry.FIELD ); + fieldsEntry.getDetails().add( fieldEntry ); + + List fieldDetails = fieldEntry.getDetails(); + fieldDetails.add( getEntry( Entry.FIELD, inputField.getName() ) ); + fieldDetails.add( getEntry( Entry.FIELD_POSITION, inputField.getPosition() ) ); + fieldDetails.add( getEntry( Entry.FIELD_LENGTH, inputField.getLength() ) ); + fieldDetails.add( getEntry( Entry.FIELD_TYPE, inputField.getType() ) ); + fieldDetails.add( getEntry( Entry.FIELD_IGNORE, inputField.isIgnored() ) ); + fieldDetails.add( getEntry( Entry.FIELD_FORMAT, inputField.getFormat() ) ); + fieldDetails.add( getEntry( Entry.FIELD_TRIM_TYPE, inputField.getTrimType() ) ); + fieldDetails.add( getEntry( Entry.FIELD_PRECISION, inputField.getPrecision() ) ); + fieldDetails.add( getEntry( Entry.FIELD_DECIMAL, inputField.getDecimalSymbol() ) ); + fieldDetails.add( getEntry( Entry.FIELD_GROUP, inputField.getGroupSymbol() ) ); + fieldDetails.add( getEntry( Entry.FIELD_CURRENCY, inputField.getCurrencySymbol() ) ); + fieldDetails.add( getEntry( Entry.FIELD_REPEAT, inputField.isRepeated() ) ); + fieldDetails.add( getEntry( Entry.FIELD_NULL_STRING, inputField.getNullString() ) ); + fieldDetails.add( getEntry( Entry.FIELD_IF_NULL, inputField.getIfNullValue() ) ); + } + } + result.add( fieldsEntry ); + + return result; + } + + public OldTextFileInputMeta getMeta() { + return meta; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileLine.java b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileLine.java new file mode 100644 index 000000000000..18ae384a88ae --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileLine.java @@ -0,0 +1,40 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import org.apache.commons.vfs2.FileObject; + +public class OldTextFileLine { + String line; + + long lineNumber; + + FileObject file; + + public OldTextFileLine( String line, long lineNumber, FileObject file ) { + super(); + this.line = line; + this.lineNumber = lineNumber; + this.file = file; + } +} diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java index ed33d0dd4db2..57f5e099d90f 100644 --- a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java @@ -22,50 +22,18 @@ package org.pentaho.di.trans.steps.textfileinput; -import java.io.BufferedInputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.apache.commons.vfs2.FileObject; -import org.pentaho.di.core.Const; -import org.pentaho.di.core.Result; -import org.pentaho.di.core.ResultFile; -import org.pentaho.di.core.compress.CompressionProvider; -import org.pentaho.di.core.compress.CompressionProviderFactory; -import org.pentaho.di.core.exception.KettleException; -import org.pentaho.di.core.exception.KettleFileException; -import org.pentaho.di.core.fileinput.FileInputList; -import org.pentaho.di.core.logging.LogChannelInterface; import org.pentaho.di.core.playlist.FilePlayListAll; import org.pentaho.di.core.playlist.FilePlayListReplay; -import org.pentaho.di.core.row.RowDataUtil; -import org.pentaho.di.core.row.RowMeta; -import org.pentaho.di.core.row.RowMetaInterface; -import org.pentaho.di.core.row.ValueMeta; -import org.pentaho.di.core.row.ValueMetaInterface; -import org.pentaho.di.core.util.StringUtil; -import org.pentaho.di.core.variables.VariableSpace; -import org.pentaho.di.core.variables.Variables; -import org.pentaho.di.core.vfs.KettleVFS; -import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; -import org.pentaho.di.trans.step.BaseStep; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepInterface; import org.pentaho.di.trans.step.StepMeta; -import org.pentaho.di.trans.step.StepMetaInterface; -import org.pentaho.di.trans.step.errorhandling.AbstractFileErrorHandler; -import org.pentaho.di.trans.step.errorhandling.CompositeFileErrorHandler; -import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; -import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerContentLineNumber; -import org.pentaho.di.trans.step.errorhandling.FileErrorHandlerMissingFiles; - +import org.pentaho.di.trans.steps.baseinput.BaseInputStep; +import org.pentaho.di.trans.steps.baseinput.IBaseInputReader; /** * Read all sorts of text files, convert them to rows and writes these to one or more output streams. @@ -73,1556 +41,50 @@ * @author Matt * @since 4-apr-2003 */ -public class TextFileInput extends BaseStep implements StepInterface { - private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! - - private static final int BUFFER_SIZE_INPUT_STREAM = 500; - - private TextFileInputMeta meta; - - private TextFileInputData data; - - private long lineNumberInFile; - - public TextFileInput( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, - Trans trans ) { - super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); - } - - public static final String getLine( LogChannelInterface log, InputStreamReader reader, int formatNr, - StringBuilder line ) throws KettleFileException { - EncodingType type = EncodingType.guessEncodingType( reader.getEncoding() ); - return getLine( log, reader, type, formatNr, line ); - } - - public static final String getLine( LogChannelInterface log, InputStreamReader reader, EncodingType encodingType, - int formatNr, StringBuilder line ) throws KettleFileException { - int c = 0; - line.setLength( 0 ); - try { - switch ( formatNr ) { - case TextFileInputMeta.FILE_FORMAT_DOS: - while ( c >= 0 ) { - c = reader.read(); - - if ( encodingType.isReturn( c ) || encodingType.isLinefeed( c ) ) { - c = reader.read(); // skip \n and \r - if ( !encodingType.isReturn( c ) && !encodingType.isLinefeed( c ) ) { - // make sure its really a linefeed or cariage return - // raise an error this is not a DOS file - // so we have pulled a character from the next line - throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.SingleLineFound" ) ); - } - return line.toString(); - } - if ( c >= 0 ) { - line.append( (char) c ); - } - } - break; - case TextFileInputMeta.FILE_FORMAT_UNIX: - while ( c >= 0 ) { - c = reader.read(); +public class TextFileInput extends BaseInputStepimplements StepInterface { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! - if ( encodingType.isLinefeed( c ) || encodingType.isReturn( c ) ) { - return line.toString(); - } - if ( c >= 0 ) { - line.append( (char) c ); - } - } - break; - case TextFileInputMeta.FILE_FORMAT_MIXED: - // in mixed mode we suppose the LF is the last char and CR is ignored - // not for MAC OS 9 but works for Mac OS X. Mac OS 9 can use UNIX-Format - while ( c >= 0 ) { - c = reader.read(); - - if ( encodingType.isLinefeed( c ) ) { - return line.toString(); - } else if ( !encodingType.isReturn( c ) ) { - if ( c >= 0 ) { - line.append( (char) c ); - } - } - } - break; - default: - break; - } - } catch ( KettleFileException e ) { - throw e; - } catch ( Exception e ) { - if ( line.length() == 0 ) { - throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ExceptionReadingLine", e - .toString() ), e ); - } - return line.toString(); - } - if ( line.length() > 0 ) { - return line.toString(); + public TextFileInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, + TransMeta transMeta, Trans trans) { + super(stepMeta, stepDataInterface, copyNr, transMeta, trans); } - return null; - } - - @Deprecated - public static final String[] guessStringsFromLine( LogChannelInterface log, String line, TextFileInputMeta inf, - String delimiter ) throws KettleException { - return guessStringsFromLine( new Variables(), log, line, inf, delimiter, StringUtil.substituteHex( inf - .getEnclosure() ), StringUtil.substituteHex( inf.getEscapeCharacter() ) ); - } - - public static final String[] guessStringsFromLine( VariableSpace space, LogChannelInterface log, String line, - TextFileInputMeta inf, String delimiter, String enclosure, String escapeCharacter ) throws KettleException { - List strings = new ArrayList(); - - String pol; // piece of line - - try { - if ( line == null ) { - return null; - } - - if ( inf.getFileType().equalsIgnoreCase( "CSV" ) ) { - - // Split string in pieces, only for CSV! - - int pos = 0; - int length = line.length(); - boolean dencl = false; - - int len_encl = ( enclosure == null ? 0 : enclosure.length() ); - int len_esc = ( escapeCharacter == null ? 0 : escapeCharacter.length() ); - - while ( pos < length ) { - int from = pos; - int next; - - boolean encl_found; - boolean contains_escaped_enclosures = false; - boolean contains_escaped_separators = false; - - // Is the field beginning with an enclosure? - // "aa;aa";123;"aaa-aaa";000;... - if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.ConvertLineToRow", line.substring( from, from + len_encl ) ) ); - } - encl_found = true; - int p = from + len_encl; - - boolean is_enclosure = - len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ) - .equalsIgnoreCase( enclosure ); - boolean is_escape = - len_esc > 0 && p + len_esc < length - && line.substring( p, p + len_esc ).equalsIgnoreCase( escapeCharacter ); - - boolean enclosure_after = false; - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ( ( is_enclosure || is_escape ) && p < length - 1 ) { - String strnext = line.substring( p + len_encl, p + 2 * len_encl ); - if ( strnext.equalsIgnoreCase( enclosure ) ) { - p++; - enclosure_after = true; - dencl = true; - - // Remember to replace them later on! - if ( is_escape ) { - contains_escaped_enclosures = true; - } - } - } - - // Look for a closing enclosure! - while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { - p++; - enclosure_after = false; - is_enclosure = - len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equals( enclosure ); - is_escape = - len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equals( escapeCharacter ); - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ( ( is_enclosure || is_escape ) && p < length - 1 ) { - - String strnext = line.substring( p + len_encl, p + 2 * len_encl ); - if ( strnext.equals( enclosure ) ) { - p++; - enclosure_after = true; - dencl = true; - - // Remember to replace them later on! - if ( is_escape ) { - contains_escaped_enclosures = true; // remember - } - } - } - } - - if ( p >= length ) { - next = p; - } else { - next = p + len_encl; - } - - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); - } - } else { - encl_found = false; - boolean found = false; - int startpoint = from; - // int tries = 1; - do { - next = line.indexOf( delimiter, startpoint ); - - // See if this position is preceded by an escape character. - if ( len_esc > 0 && next - len_esc > 0 ) { - String before = line.substring( next - len_esc, next ); - - if ( escapeCharacter.equals( before ) ) { - // take the next separator, this one is escaped... - startpoint = next + 1; - // tries++; - contains_escaped_separators = true; - } else { - found = true; - } - } else { - found = true; - } - } while ( !found && next >= 0 ); - } - if ( next == -1 ) { - next = length; - } - - if ( encl_found ) { - pol = line.substring( from + len_encl, next - len_encl ); - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); - } - } else { - pol = line.substring( from, next ); - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); - } - } - - if ( dencl ) { - StringBuilder sbpol = new StringBuilder( pol ); - int idx = sbpol.indexOf( enclosure + enclosure ); - while ( idx >= 0 ) { - sbpol.delete( idx, idx + enclosure.length() ); - idx = sbpol.indexOf( enclosure + enclosure ); - } - pol = sbpol.toString(); - } - - // replace the escaped enclosures with enclosures... - if ( contains_escaped_enclosures ) { - String replace = escapeCharacter + enclosure; - String replaceWith = enclosure; - - pol = Const.replace( pol, replace, replaceWith ); - } - - // replace the escaped separators with separators... - if ( contains_escaped_separators ) { - String replace = escapeCharacter + delimiter; - String replaceWith = delimiter; - - pol = Const.replace( pol, replace, replaceWith ); - } - - // Now add pol to the strings found! - strings.add( pol ); - - pos = next + delimiter.length(); - } - if ( pos == length ) { - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); - } - strings.add( "" ); - } - } else { - // Fixed file format: Simply get the strings at the required positions... - for ( int i = 0; i < inf.getInputFields().length; i++ ) { - TextFileInputField field = inf.getInputFields()[i]; - - int length = line.length(); - - if ( field.getPosition() + field.getLength() <= length ) { - strings.add( line.substring( field.getPosition(), field.getPosition() + field.getLength() ) ); - } else { - if ( field.getPosition() < length ) { - strings.add( line.substring( field.getPosition() ) ); - } else { - strings.add( "" ); - } - } - } - } - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e - .toString() ), e ); + @Override + protected IBaseInputReader createReader(TextFileInputMeta meta, TextFileInputData data, FileObject file) + throws Exception { + return new TextFileInputReader(this, meta, data, file, log); } - return strings.toArray( new String[strings.size()] ); - } - - public static final String[] convertLineToStrings( LogChannelInterface log, String line, InputFileMetaInterface inf, - String delimiter, String enclosure, String escapeCharacters ) throws KettleException { - String[] strings = new String[inf.getInputFields().length]; - int fieldnr; - - String pol; // piece of line - - try { - if ( line == null ) { - return null; - } - - if ( inf.getFileType().equalsIgnoreCase( "CSV" ) ) { - // Split string in pieces, only for CSV! - - fieldnr = 0; - int pos = 0; - int length = line.length(); - boolean dencl = false; - - int len_encl = ( enclosure == null ? 0 : enclosure.length() ); - int len_esc = ( escapeCharacters == null ? 0 : escapeCharacters.length() ); - - while ( pos < length ) { - int from = pos; - int next; - - boolean encl_found; - boolean contains_escaped_enclosures = false; - boolean contains_escaped_separators = false; - - // Is the field beginning with an enclosure? - // "aa;aa";123;"aaa-aaa";000;... - if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.Encloruse", line.substring( from, from + len_encl ) ) ); - } - encl_found = true; - int p = from + len_encl; - - boolean is_enclosure = - len_encl > 0 && p + len_encl < length - && line.substring( p, p + len_encl ).equalsIgnoreCase( enclosure ); - boolean is_escape = - len_esc > 0 && p + len_esc < length - && line.substring( p, p + len_esc ).equalsIgnoreCase( inf.getEscapeCharacter() ); - - boolean enclosure_after = false; - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ( ( is_enclosure || is_escape ) && p < length - 1 ) { - String strnext = line.substring( p + len_encl, p + 2 * len_encl ); - if ( strnext.equalsIgnoreCase( enclosure ) ) { - p++; - enclosure_after = true; - dencl = true; - - // Remember to replace them later on! - if ( is_escape ) { - contains_escaped_enclosures = true; - } - } - } - - // Look for a closing enclosure! - while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { - p++; - enclosure_after = false; - is_enclosure = - len_encl > 0 && p + len_encl < length - && line.substring( p, p + len_encl ).equals( enclosure ); - is_escape = - len_esc > 0 && p + len_esc < length - && line.substring( p, p + len_esc ).equals( inf.getEscapeCharacter() ); - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ( ( is_enclosure || is_escape ) && p < length - 1 ) { - - String strnext = line.substring( p + len_encl, p + 2 * len_encl ); - if ( strnext.equals( enclosure ) ) { - p++; - enclosure_after = true; - dencl = true; - - // Remember to replace them later on! - if ( is_escape ) { - contains_escaped_enclosures = true; // remember - } - } - } - } - - if ( p >= length ) { - next = p; - } else { - next = p + len_encl; - } - - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); - } - } else { - encl_found = false; - boolean found = false; - int startpoint = from; - // int tries = 1; - do { - next = line.indexOf( delimiter, startpoint ); - - // See if this position is preceded by an escape character. - if ( len_esc > 0 && next - len_esc > 0 ) { - String before = line.substring( next - len_esc, next ); - - if ( inf.getEscapeCharacter().equals( before ) ) { - // take the next separator, this one is escaped... - startpoint = next + 1; - // tries++; - contains_escaped_separators = true; - } else { - found = true; - } - } else { - found = true; - } - } while ( !found && next >= 0 ); - } - if ( next == -1 ) { - next = length; - } - - if ( encl_found && ( ( from + len_encl ) <= ( next - len_encl ) ) ) { - pol = line.substring( from + len_encl, next - len_encl ); - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); - } - } else { - pol = line.substring( from, next ); - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); - } - } - - if ( dencl && Const.isEmpty( inf.getEscapeCharacter() ) ) { - StringBuilder sbpol = new StringBuilder( pol ); - int idx = sbpol.indexOf( enclosure + enclosure ); - while ( idx >= 0 ) { - sbpol.delete( idx, idx + enclosure.length() ); - idx = sbpol.indexOf( enclosure + enclosure ); - } - pol = sbpol.toString(); - } - - // replace the escaped enclosures with enclosures... - if ( contains_escaped_enclosures ) { - String replace = inf.getEscapeCharacter() + enclosure; - String replaceWith = enclosure; - - pol = Const.replace( pol, replace, replaceWith ); - } - - // replace the escaped separators with separators... - if ( contains_escaped_separators ) { - String replace = inf.getEscapeCharacter() + delimiter; - String replaceWith = delimiter; - - pol = Const.replace( pol, replace, replaceWith ); - } - - // Now add pol to the strings found! - try { - strings[fieldnr] = pol; - } catch ( ArrayIndexOutOfBoundsException e ) { - // In case we didn't allocate enough space. - // This happens when you have less header values specified than there are actual values in the rows. - // As this is "the exception" we catch and resize here. - // - String[] newStrings = new String[strings.length]; - for ( int x = 0; x < strings.length; x++ ) { - newStrings[x] = strings[x]; - } - strings = newStrings; - } - - pos = next + delimiter.length(); - fieldnr++; - } - if ( pos == length ) { - if ( log.isRowLevel() ) { - log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages - .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); - } - if ( fieldnr < strings.length ) { - strings[fieldnr] = Const.EMPTY_STRING; - } - fieldnr++; - } - } else { - // Fixed file format: Simply get the strings at the required positions... - for ( int i = 0; i < inf.getInputFields().length; i++ ) { - TextFileInputField field = inf.getInputFields()[i]; - - int length = line.length(); - - if ( field.getPosition() + field.getLength() <= length ) { - strings[i] = line.substring( field.getPosition(), field.getPosition() + field.getLength() ); - } else { - if ( field.getPosition() < length ) { - strings[i] = line.substring( field.getPosition() ); - } else { - strings[i] = ""; - } - } - } - } - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e - .toString() ), e ); - } - - return strings; - } - - /** - * @deprecated Use {@link #convertLineToRow(LogChannelInterface, TextFileLine, - * InputFileMetaInterface, Object[], int, RowMetaInterface,RowMetaInterface, - * String, long, String, String, String, FileErrorHandler, boolean, boolean, - * boolean, boolean, boolean, boolean, boolean, boolean, String, String, boolean, - * Date, String, String, String, long)} instead. - */ - @Deprecated - public static final Object[] convertLineToRow( LogChannelInterface log, TextFileLine textFileLine, - InputFileMetaInterface info, RowMetaInterface outputRowMeta, RowMetaInterface convertRowMeta, String fname, - long rowNr, String delimiter, FileErrorHandler errorHandler, boolean addShortFilename, boolean addExtension, - boolean addPath, boolean addSize, boolean addIsHidden, boolean addLastModificationDate, boolean addUri, - boolean addRootUri, String shortFilename, String path, boolean hidden, Date modificationDateTime, String uri, - String rooturi, String extension, long size ) throws KettleException { - return convertLineToRow( log, textFileLine, info, null, 0, outputRowMeta, convertRowMeta, fname, rowNr, delimiter, - StringUtil.substituteHex( info.getEnclosure() ), StringUtil.substituteHex( info.getEscapeCharacter() ), - errorHandler, addShortFilename, addExtension, addPath, addSize, addIsHidden, addLastModificationDate, addUri, - addRootUri, shortFilename, path, hidden, modificationDateTime, uri, rooturi, extension, size ); - } - - public static final Object[] convertLineToRow( LogChannelInterface log, TextFileLine textFileLine, - InputFileMetaInterface info, Object[] passThruFields, int nrPassThruFields, RowMetaInterface outputRowMeta, - RowMetaInterface convertRowMeta, String fname, long rowNr, String delimiter, String enclosure, - String escapeCharacter, FileErrorHandler errorHandler, boolean addShortFilename, boolean addExtension, - boolean addPath, boolean addSize, boolean addIsHidden, boolean addLastModificationDate, boolean addUri, - boolean addRootUri, String shortFilename, String path, boolean hidden, Date modificationDateTime, String uri, - String rooturi, String extension, long size ) throws KettleException { - if ( textFileLine == null || textFileLine.line == null ) { - return null; - } - - Object[] r = RowDataUtil.allocateRowData( outputRowMeta.size() ); // over-allocate a bit in the row producing - // steps... - - int nrfields = info.getInputFields().length; - int fieldnr; - - Long errorCount = null; - if ( info.isErrorIgnored() && info.getErrorCountField() != null && info.getErrorCountField().length() > 0 ) { - errorCount = new Long( 0L ); - } - String errorFields = null; - if ( info.isErrorIgnored() && info.getErrorFieldsField() != null && info.getErrorFieldsField().length() > 0 ) { - errorFields = ""; - } - String errorText = null; - if ( info.isErrorIgnored() && info.getErrorTextField() != null && info.getErrorTextField().length() > 0 ) { - errorText = ""; - } - - try { - // System.out.println("Convertings line to string ["+line+"]"); - String[] strings = convertLineToStrings( log, textFileLine.line, info, delimiter, enclosure, escapeCharacter ); - int shiftFields = ( passThruFields == null ? 0 : nrPassThruFields ); - for ( fieldnr = 0; fieldnr < nrfields; fieldnr++ ) { - TextFileInputField f = info.getInputFields()[fieldnr]; - int valuenr = shiftFields + fieldnr; - ValueMetaInterface valueMeta = outputRowMeta.getValueMeta( valuenr ); - ValueMetaInterface convertMeta = convertRowMeta.getValueMeta( valuenr ); - - Object value; - - String nullif = fieldnr < nrfields ? f.getNullString() : ""; - String ifnull = fieldnr < nrfields ? f.getIfNullValue() : ""; - int trim_type = fieldnr < nrfields ? f.getTrimType() : ValueMetaInterface.TRIM_TYPE_NONE; - - if ( fieldnr < strings.length ) { - String pol = strings[fieldnr]; - try { - value = valueMeta.convertDataFromString( pol, convertMeta, nullif, ifnull, trim_type ); - } catch ( Exception e ) { - // OK, give some feedback! - String message = - BaseMessages.getString( PKG, "TextFileInput.Log.CoundNotParseField", valueMeta.toStringMeta(), - "" + pol, valueMeta.getConversionMask(), "" + rowNr ); - - if ( info.isErrorIgnored() ) { - log.logDetailed( fname, BaseMessages.getString( PKG, "TextFileInput.Log.Warning" ) + ": " + message - + " : " + e.getMessage() ); - - value = null; - - if ( errorCount != null ) { - errorCount = new Long( errorCount.longValue() + 1L ); - } - if ( errorFields != null ) { - StringBuilder sb = new StringBuilder( errorFields ); - if ( sb.length() > 0 ) { - sb.append( "\t" ); // TODO document this change - } - sb.append( valueMeta.getName() ); - errorFields = sb.toString(); - } - if ( errorText != null ) { - StringBuilder sb = new StringBuilder( errorText ); - if ( sb.length() > 0 ) { - sb.append( Const.CR ); - } - sb.append( message ); - errorText = sb.toString(); - } - if ( errorHandler != null ) { - errorHandler.handleLineError( textFileLine.lineNumber, AbstractFileErrorHandler.NO_PARTS ); - } - - if ( info.isErrorLineSkipped() ) { - r = null; // compensates for stmt: r.setIgnore(); - } - } else { - throw new KettleException( message, e ); - } - } + @Override + public boolean init() { + Date replayDate = getTrans().getReplayDate(); + if (replayDate == null) { + data.filePlayList = FilePlayListAll.INSTANCE; } else { - // No data found: TRAILING NULLCOLS: add null value... - value = null; - } - - // Now add value to the row (if we're not skipping the row) - if ( r != null ) { - r[valuenr] = value; + data.filePlayList = new FilePlayListReplay(replayDate, + meta.errorHandling.lineNumberFilesDestinationDirectory, + meta.errorHandling.lineNumberFilesExtension, + meta.errorHandling.errorFilesDestinationDirectory, meta.errorHandling.errorFilesExtension, + meta.content.encoding); } - } - // none of this applies if we're skipping the row - if ( r != null ) { - // Support for trailing nullcols! - // Should be OK at allocation time, but it doesn't hurt :-) - if ( fieldnr < nrfields ) { - for ( int i = fieldnr; i < info.getInputFields().length; i++ ) { - r[shiftFields + i] = null; - } - } + data.filterProcessor = new TextFileFilterProcessor(meta.getFilter()); - // Add the error handling fields... - int index = shiftFields + nrfields; - if ( errorCount != null ) { - r[index] = errorCount; - index++; - } - if ( errorFields != null ) { - r[index] = errorFields; - index++; - } - if ( errorText != null ) { - r[index] = errorText; - index++; - } + // calculate the file format type in advance so we can use a switch + data.fileFormatType = meta.getFileFormatTypeNr(); - // Possibly add a filename... - if ( info.includeFilename() ) { - r[index] = fname; - index++; - } + // calculate the file type in advance CSV or Fixed? + data.fileType = meta.getFileTypeNr(); - // Possibly add a row number... - if ( info.includeRowNumber() ) { - r[index] = new Long( rowNr ); - index++; - } + // Handle the possibility of a variable substitution + data.separator = environmentSubstitute(meta.content.separator); + data.enclosure = environmentSubstitute(meta.content.enclosure); + data.escapeCharacter = environmentSubstitute(meta.content.escapeCharacter); - // Possibly add short filename... - if ( addShortFilename ) { - r[index] = shortFilename; - index++; - } - // Add Extension - if ( addExtension ) { - r[index] = extension; - index++; - } - // add path - if ( addPath ) { - r[index] = path; - index++; - } - // Add Size - if ( addSize ) { - r[index] = new Long( size ); - index++; - } - // add Hidden - if ( addIsHidden ) { - r[index] = hidden; - index++; - } - // Add modification date - if ( addLastModificationDate ) { - r[index] = modificationDateTime; - index++; - } - // Add Uri - if ( addUri ) { - r[index] = uri; - index++; - } - // Add RootUri - if ( addRootUri ) { - r[index] = rooturi; - index++; - } - } // End if r != null - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLineText" ), e ); + return true; } - if ( passThruFields != null ) { - // Simply add all fields from source files step - for ( int i = 0; i < nrPassThruFields; i++ ) { - r[i] = passThruFields[i]; - } + public boolean isWaitingForData() { + return true; } - - return r; - - } - - @Override - public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { - data = (TextFileInputData) sdi; - meta = (TextFileInputMeta) smi; - Object[] r = null; - boolean retval = true; - boolean putrow = false; - - if ( first ) { // we just got started - - first = false; - - data.outputRowMeta = new RowMeta(); - RowMetaInterface[] infoStep = null; - - if ( meta.isAcceptingFilenames() ) { - // Read the files from the specified input stream... - // - data.getFiles().getFiles().clear(); - - int idx = -1; - data.rowSet = findInputRowSet( meta.getAcceptingStepName() ); - - Object[] fileRow = getRowFrom( data.rowSet ); - while ( fileRow != null ) { - RowMetaInterface prevInfoFields = data.rowSet.getRowMeta(); - if ( idx < 0 ) { - if ( meta.isPassingThruFields() ) { - data.passThruFields = new HashMap(); - infoStep = new RowMetaInterface[] { prevInfoFields }; - data.nrPassThruFields = prevInfoFields.size(); - } - idx = prevInfoFields.indexOfValue( meta.getAcceptingField() ); - if ( idx < 0 ) { - logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.UnableToFindFilenameField", meta - .getAcceptingField() ) ); - setErrors( getErrors() + 1 ); - stopAll(); - return false; - } - } - String fileValue = prevInfoFields.getString( fileRow, idx ); - try { - FileObject fileObject = KettleVFS.getFileObject( fileValue, getTransMeta() ); - data.getFiles().addFile( fileObject ); - if ( meta.isPassingThruFields() ) { - data.passThruFields.put( fileObject, fileRow ); - } - } catch ( KettleFileException e ) { - logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.UnableToCreateFileObject", fileValue ), e ); - } - - // Grab another row - fileRow = getRowFrom( data.rowSet ); - } - - if ( data.getFiles().nrOfFiles() == 0 ) { - if ( log.isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "TextFileInput.Log.Error.NoFilesSpecified" ) ); - } - setOutputDone(); - return false; - } - } - - // // get the metadata populated. Simple and easy. - meta.getFields( data.outputRowMeta, getStepname(), infoStep, null, this, repository, metaStore ); - // Create convert meta-data objects that will contain Date & Number formatters - // - data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); - - handleMissingFiles(); - - // Open the first file & read the required rows in the buffer, stop - // if it fails and not set to skip bad files... - if ( !openNextFile() ) { - if ( failAfterBadFile( null ) ) { - closeLastFile(); - setOutputDone(); - return false; - } - } - - // Count the number of repeat fields... - for ( int i = 0; i < meta.getInputFields().length; i++ ) { - if ( meta.getInputFields()[i].isRepeated() ) { - data.nr_repeats++; - } - } - } else { - if ( !data.doneReading ) { - int repeats = 1; - if ( meta.isLineWrapped() ) { - repeats = meta.getNrWraps() > 0 ? meta.getNrWraps() : repeats; - } - - if ( !data.doneWithHeader && data.headerLinesRead == 0 ) { - // We are just starting to read header lines, read them all - repeats += meta.getNrHeaderLines() + 1; - } - - // Read a number of lines... - for ( int i = 0; i < repeats && !data.doneReading; i++ ) { - tryToReadLine( true ); - } - } - } - - /* - * If the buffer is empty: open the next file. (if nothing in there, open the next, etc.) - */ - while ( data.lineBuffer.size() == 0 ) { - if ( !openNextFile() ) // Open fails: done processing unless set to skip bad files - { - if ( failAfterBadFile( null ) ) { - closeLastFile(); - setOutputDone(); // signal end to receiver(s) - return false; - } // else will continue until can open - } - } - - /* - * Take the first line available in the buffer & remove the line from the buffer - */ - TextFileLine textLine = data.lineBuffer.get( 0 ); - incrementLinesInput(); - lineNumberInFile++; - - data.lineBuffer.remove( 0 ); - - if ( meta.isLayoutPaged() ) { - /* - * Different rules apply: on each page: a header a number of data lines a footer - */ - if ( !data.doneWithHeader && data.pageLinesRead == 0 ) // We are reading header lines - { - if ( log.isRowLevel() ) { - logRowlevel( "P-HEADER (" + data.headerLinesRead + ") : " + textLine.line ); - } - data.headerLinesRead++; - if ( data.headerLinesRead >= meta.getNrHeaderLines() ) { - data.doneWithHeader = true; - } - } else { - // data lines or footer on a page - - if ( data.pageLinesRead < meta.getNrLinesPerPage() ) { - // See if we are dealing with wrapped lines: - if ( meta.isLineWrapped() ) { - for ( int i = 0; i < meta.getNrWraps(); i++ ) { - String extra = ""; - if ( data.lineBuffer.size() > 0 ) { - extra = data.lineBuffer.get( 0 ).line; - data.lineBuffer.remove( 0 ); - } - textLine.line += extra; - } - } - - if ( log.isRowLevel() ) { - logRowlevel( "P-DATA: " + textLine.line ); - } - // Read a normal line on a page of data. - data.pageLinesRead++; - data.lineInFile++; - long useNumber = meta.isRowNumberByFile() ? data.lineInFile : getLinesWritten() + 1; - r = - convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, data.nrPassThruFields, - data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, data.separator, data.enclosure, - data.escapeCharacter, data.dataErrorLineHandler, data.addShortFilename, data.addExtension, - data.addPath, data.addSize, data.addIsHidden, data.addLastModificationDate, data.addUri, - data.addRootUri, data.shortFilename, data.path, data.hidden, data.lastModificationDateTime, - data.uriName, data.rootUriName, data.extension, data.size ); - if ( r != null ) { - putrow = true; - } - - // Possible fix for bug PDI-1121 - paged layout header and line count off by 1 - // We need to reset these BEFORE the next header line is read, so that it - // is treated as a header ... obviously, only if there is no footer, and we are - // done reading data. - if ( !meta.hasFooter() && ( data.pageLinesRead == meta.getNrLinesPerPage() ) ) { - /* - * OK, we are done reading the footer lines, start again on 'next page' with the header - */ - data.doneWithHeader = false; - data.headerLinesRead = 0; - data.pageLinesRead = 0; - data.footerLinesRead = 0; - if ( log.isRowLevel() ) { - logRowlevel( "RESTART PAGE" ); - } - } - } else { - // done reading the data lines, skip the footer lines - - if ( meta.hasFooter() && data.footerLinesRead < meta.getNrFooterLines() ) { - if ( log.isRowLevel() ) { - logRowlevel( "P-FOOTER: " + textLine.line ); - } - data.footerLinesRead++; - } - - if ( !meta.hasFooter() || data.footerLinesRead >= meta.getNrFooterLines() ) { - /* - * OK, we are done reading the footer lines, start again on 'next page' with the header - */ - data.doneWithHeader = false; - data.headerLinesRead = 0; - data.pageLinesRead = 0; - data.footerLinesRead = 0; - if ( log.isRowLevel() ) { - logRowlevel( "RESTART PAGE" ); - } - } - } - } - } else { - // A normal data line, can also be a header or a footer line - - if ( !data.doneWithHeader ) { // We are reading header lines - - data.headerLinesRead++; - if ( data.headerLinesRead >= meta.getNrHeaderLines() ) { - data.doneWithHeader = true; - } - } else { - /* - * IF we are done reading and we have a footer AND the number of lines in the buffer is smaller then the number - * of footer lines THEN we can remove the remaining rows from the buffer: they are all footer rows. - */ - if ( data.doneReading && meta.hasFooter() && data.lineBuffer.size() < meta.getNrFooterLines() ) { - data.lineBuffer.clear(); - } else { - // Not yet a footer line: it's a normal data line. - - // See if we are dealing with wrapped lines: - if ( meta.isLineWrapped() ) { - for ( int i = 0; i < meta.getNrWraps(); i++ ) { - String extra = ""; - if ( data.lineBuffer.size() > 0 ) { - extra = data.lineBuffer.get( 0 ).line; - data.lineBuffer.remove( 0 ); - } else { - tryToReadLine( true ); - if ( !data.lineBuffer.isEmpty() ) { - extra = data.lineBuffer.remove( 0 ).line; - } - } - textLine.line += extra; - } - } - if ( data.filePlayList.isProcessingNeeded( textLine.file, textLine.lineNumber, - AbstractFileErrorHandler.NO_PARTS ) ) { - data.lineInFile++; - long useNumber = meta.isRowNumberByFile() ? data.lineInFile : getLinesWritten() + 1; - r = - convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, data.nrPassThruFields, - data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, data.separator, data.enclosure, - data.escapeCharacter, data.dataErrorLineHandler, data.addShortFilename, data.addExtension, - data.addPath, data.addSize, data.addIsHidden, data.addLastModificationDate, data.addUri, - data.addRootUri, data.shortFilename, data.path, data.hidden, data.lastModificationDateTime, - data.uriName, data.rootUriName, data.extension, data.size ); - if ( r != null ) { - if ( log.isRowLevel() ) { - logRowlevel( "Found data row: " + data.outputRowMeta.getString( r ) ); - } - putrow = true; - } - } else { - putrow = false; - } - } - } - } - - if ( putrow && r != null ) { - // See if the previous values need to be repeated! - if ( data.nr_repeats > 0 ) { - if ( data.previous_row == null ) { // First invocation... - - data.previous_row = data.outputRowMeta.cloneRow( r ); - } else { - // int repnr = 0; - for ( int i = 0; i < meta.getInputFields().length; i++ ) { - if ( meta.getInputFields()[i].isRepeated() ) { - if ( r[i] == null ) { - // if it is empty: take the previous value! - - r[i] = data.previous_row[i]; - } else { - // not empty: change the previous_row entry! - - data.previous_row[i] = r[i]; - } - // repnr++; - } - } - } - } - - if ( log.isRowLevel() ) { - logRowlevel( "Putting row: " + data.outputRowMeta.getString( r ) ); - } - putRow( data.outputRowMeta, r ); - - if ( getLinesInput() >= meta.getRowLimit() && meta.getRowLimit() > 0 ) { - closeLastFile(); - setOutputDone(); // signal end to receiver(s) - return false; - } - } - - if ( checkFeedback( getLinesInput() ) ) { - if ( log.isBasic() ) { - logBasic( "linenr " + getLinesInput() ); - } - } - - return retval; - } - - /** - * - * @param errorMsg - * Message to send to rejected row if enabled - * @return If should stop processing after having problems with a file - */ - private boolean failAfterBadFile( String errorMsg ) { - - if ( getStepMeta().isDoingErrorHandling() && data.filename != null - && !data.rejectedFiles.containsKey( data.filename ) ) { - data.rejectedFiles.put( data.filename, true ); - rejectCurrentFile( errorMsg ); - } - - return !meta.isErrorIgnored() || !meta.isSkipBadFiles() || data.isLastFile; - } - - /** - * Send file name and/or error message to error output - * - * @param errorMsg - * Message to send to rejected row if enabled - */ - private void rejectCurrentFile( String errorMsg ) { - if ( StringUtils.isNotBlank( meta.getFileErrorField() ) - || StringUtils.isNotBlank( meta.getFileErrorMessageField() ) ) { - RowMetaInterface rowMeta = getInputRowMeta(); - if ( rowMeta == null ) { - rowMeta = new RowMeta(); - } - - int errorFileIndex = - ( StringUtils.isBlank( meta.getFileErrorField() ) ) ? -1 : addValueMeta( rowMeta, this - .environmentSubstitute( meta.getFileErrorField() ) ); - - int errorMessageIndex = - StringUtils.isBlank( meta.getFileErrorMessageField() ) ? -1 : addValueMeta( rowMeta, this - .environmentSubstitute( meta.getFileErrorMessageField() ) ); - - try { - Object[] rowData = getRow(); - if ( rowData == null ) { - rowData = RowDataUtil.allocateRowData( rowMeta.size() ); - } - - if ( errorFileIndex >= 0 ) { - rowData[errorFileIndex] = data.filename; - } - if ( errorMessageIndex >= 0 ) { - rowData[errorMessageIndex] = errorMsg; - } - - putError( rowMeta, rowData, getErrors(), data.filename, null, "ERROR_CODE" ); - } catch ( Exception e ) { - logError( "Error sending error row", e ); - } - } - } - - /** - * Adds String value meta with given name if not present and returns index - * - * @param rowMeta - * @param fieldName - * @return Index in row meta of value meta with fieldName - */ - private int addValueMeta( RowMetaInterface rowMeta, String fieldName ) { - ValueMetaInterface valueMeta = new ValueMeta( fieldName, ValueMetaInterface.TYPE_STRING ); - valueMeta.setOrigin( getStepname() ); - // add if doesn't exist - int index = -1; - if ( !rowMeta.exists( valueMeta ) ) { - index = rowMeta.size(); - rowMeta.addValueMeta( valueMeta ); - } else { - index = rowMeta.indexOfValue( fieldName ); - } - return index; - } - - /** - * Check if the line should be taken. - * - * @param line - * @param isFilterLastLine - * (dummy input param, only set when return value is false) - * @return true when the line should be taken (when false, isFilterLastLine will be set) - */ - private boolean checkFilterRow( String line, boolean isFilterLastLine ) { - boolean filterOK = true; - - // check for noEmptyLines - if ( meta.noEmptyLines() && line.length() == 0 ) { - filterOK = false; - } else { - // check the filters - filterOK = data.filterProcessor.doFilters( line ); - if ( !filterOK ) { - if ( data.filterProcessor.isStopProcessing() ) { - data.doneReading = true; - } - } - } - - return filterOK; - } - - private void handleMissingFiles() throws KettleException { - List nonExistantFiles = data.getFiles().getNonExistantFiles(); - - if ( nonExistantFiles.size() != 0 ) { - String message = FileInputList.getRequiredFilesDescription( nonExistantFiles ); - if ( log.isBasic() ) { - log.logBasic( "Required files", "WARNING: Missing " + message ); - } - if ( meta.isErrorIgnored() ) { - for ( FileObject fileObject : nonExistantFiles ) { - data.dataErrorLineHandler.handleNonExistantFile( fileObject ); - } - } else { - throw new KettleException( "Following required files are missing: " + message ); - } - } - - List nonAccessibleFiles = data.getFiles().getNonAccessibleFiles(); - if ( nonAccessibleFiles.size() != 0 ) { - String message = FileInputList.getRequiredFilesDescription( nonAccessibleFiles ); - if ( log.isBasic() ) { - log.logBasic( "Required files", "WARNING: Not accessible " + message ); - } - if ( meta.isErrorIgnored() ) { - for ( FileObject fileObject : nonAccessibleFiles ) { - data.dataErrorLineHandler.handleNonAccessibleFile( fileObject ); - } - } else { - throw new KettleException( "Following required files are not accessible: " + message ); - } - } - } - - private boolean closeLastFile() { - try { - // Close previous file! - if ( data.filename != null ) { - // Increment the lines updated to reflect another file has been finished. - // This allows us to give a state of progress in the run time metrics - incrementLinesUpdated(); - /* - * } else if ( sFileCompression != null && sFileCompression.equals( "Snappy" ) && data.sis != null ) { - * data.sis.close(); } - */ - data.in.close(); - data.isr.close(); - data.filename = null; // send it down the next time. - if ( data.file != null ) { - data.file.close(); - data.file = null; - } - } - data.dataErrorLineHandler.close(); - } catch ( Exception e ) { - String errorMsg = "Couldn't close file : " + data.file.getName().getFriendlyURI() + " --> " + e.toString(); - logError( errorMsg ); - if ( failAfterBadFile( errorMsg ) ) { // ( !meta.isSkipBadFiles() || data.isLastFile ){ - stopAll(); - } - setErrors( getErrors() + 1 ); - return false; - } // finally { - // This is for bug #5797 : it tries to assure that the file handle - // is actually freed/garbarge collected. - // XXX deinspanjer 2009-07-07: I'm stubbing this out. The bug was ancient and it is worth reevaluating - // to avoid the performance hit of a System GC on every file close - // System.gc(); - // } - - return !data.isLastFile; - } - - private boolean openNextFile() { - - try { - lineNumberInFile = 0; - if ( !closeLastFile() && failAfterBadFile( null ) ) { - return false; // (!meta.isSkipBadFiles() || data.isLastFile) ) return false; - } - - if ( data.getFiles().nrOfFiles() == 0 ) { - return false; - } - - // Is this the last file? - data.isLastFile = ( data.filenr == data.getFiles().nrOfFiles() - 1 ); - data.file = data.getFiles().getFile( data.filenr ); - data.filename = KettleVFS.getFilename( data.file ); - - // Move file pointer ahead! - data.filenr++; - - // Add additional fields? - if ( data.addShortFilename ) { - data.shortFilename = data.file.getName().getBaseName(); - } - if ( data.addPath ) { - data.path = KettleVFS.getFilename( data.file.getParent() ); - } - if ( data.addIsHidden ) { - data.hidden = data.file.isHidden(); - } - if ( data.addExtension ) { - data.extension = data.file.getName().getExtension(); - } - if ( data.addLastModificationDate ) { - data.lastModificationDateTime = new Date( data.file.getContent().getLastModifiedTime() ); - } - if ( data.addUri ) { - data.uriName = data.file.getName().getURI(); - } - if ( data.addRootUri ) { - data.rootUriName = data.file.getName().getRootURI(); - } - if ( data.addSize ) { - data.size = new Long( data.file.getContent().getSize() ); - } - data.lineInFile = 0; - if ( meta.isPassingThruFields() ) { - data.currentPassThruFieldsRow = data.passThruFields.get( data.file ); - } - - // Add this files to the result of this transformation. - // - if ( meta.isAddResultFile() ) { - ResultFile resultFile = - new ResultFile( ResultFile.FILE_TYPE_GENERAL, data.file, getTransMeta().getName(), toString() ); - resultFile.setComment( "File was read by an Text File input step" ); - addResultFile( resultFile ); - } - if ( log.isBasic() ) { - logBasic( "Opening file: " + data.file.getName().getFriendlyURI() ); - } - - CompressionProvider provider = - CompressionProviderFactory.getInstance().getCompressionProviderByName( meta.getFileCompression() ); - - data.in = provider.createInputStream( KettleVFS.getInputStream( data.file ) ); - data.dataErrorLineHandler.handleFile( data.file ); - data.in.nextEntry(); - - if ( log.isDetailed() ) { - logDetailed( "This is a compressed file being handled by the " + provider.getName() + " provider" ); - } - - if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { - data.isr = - new InputStreamReader( new BufferedInputStream( data.in, BUFFER_SIZE_INPUT_STREAM ), meta.getEncoding() ); - } else { - data.isr = new InputStreamReader( new BufferedInputStream( data.in, BUFFER_SIZE_INPUT_STREAM ) ); - } - - String encoding = data.isr.getEncoding(); - data.encodingType = EncodingType.guessEncodingType( encoding ); - - // ///////////////////////////////////////////////////////////////////////////// - // Read the first lines... - - /* - * Keep track of the status of the file: are there any lines left to read? - */ - data.doneReading = false; - - /* - * OK, read a number of lines in the buffer: The header rows The nr rows in the page : optional The footer rows - */ - int bufferSize = 1; - bufferSize += meta.hasHeader() ? meta.getNrHeaderLines() : 0; - bufferSize += meta.isLayoutPaged() - ? meta.getNrLinesPerPage() * ( Math.max( 0, meta.getNrWraps() ) + 1 ) - : Math.max( 0, meta.getNrWraps() ); // it helps when we have wrapped input w/o header - - bufferSize += meta.hasFooter() ? meta.getNrFooterLines() : 0; - - // See if we need to skip the document header lines... - if ( meta.isLayoutPaged() ) { - for ( int i = 0; i < meta.getNrLinesDocHeader(); i++ ) { - // Just skip these... - getLine( log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); // header and - // footer: not - // wrapped - lineNumberInFile++; - } - } - - for ( int i = 0; i < bufferSize && !data.doneReading; i++ ) { - boolean wasNotFiltered = tryToReadLine( !meta.hasHeader() || i >= meta.getNrHeaderLines() ); - if ( !wasNotFiltered ) { - // grab another line, this one got filtered - bufferSize++; - } - } - - // Reset counters etc. - data.headerLinesRead = 0; - data.footerLinesRead = 0; - data.pageLinesRead = 0; - - // Set a flags - data.doneWithHeader = !meta.hasHeader(); - } catch ( Exception e ) { - String errorMsg = - "Couldn't open file #" + data.filenr + " : " + data.file.getName().getFriendlyURI() + " --> " + e.toString(); - logError( errorMsg ); - if ( failAfterBadFile( errorMsg ) ) { // !meta.isSkipBadFiles()) stopAll(); - stopAll(); - } - setErrors( getErrors() + 1 ); - return false; - } - return true; - } - - private boolean tryToReadLine( boolean applyFilter ) throws KettleFileException { - String line; - line = getLine( log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); - if ( line != null ) { - // when there is no header, check the filter for the first line - if ( applyFilter ) { - // Filter row? - boolean isFilterLastLine = false; - boolean filterOK = checkFilterRow( line, isFilterLastLine ); - if ( filterOK ) { - data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the - // line buffer... - } else { - return false; - } - } else { // don't checkFilterRow - - if ( !meta.noEmptyLines() || line.length() != 0 ) { - data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the line - // buffer... - } - } - } else { - data.doneReading = true; - } - return true; - } - - @Override - public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { - meta = (TextFileInputMeta) smi; - data = (TextFileInputData) sdi; - - if ( super.init( smi, sdi ) ) { - initErrorHandling(); - initReplayFactory(); - - data.setFiles( meta.getTextFileList( this ) ); - data.filterProcessor = new TextFileFilterProcessor( meta.getFilter() ); - - // If there are missing files, - // fail if we don't ignore errors - // - Result previousResult = getTrans().getPreviousResult(); - Map resultFiles = ( previousResult != null ) ? previousResult.getResultFiles() : null; - - if ( ( previousResult == null || resultFiles == null || resultFiles.size() == 0 ) - && data.getFiles().nrOfMissingFiles() > 0 && !meta.isAcceptingFilenames() && !meta.isErrorIgnored() ) { - logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.NoFilesSpecified" ) ); - return false; - } - - String clusterSize = getVariable( Const.INTERNAL_VARIABLE_CLUSTER_SIZE ); - if ( !Const.isEmpty( clusterSize ) && Integer.valueOf( clusterSize ) > 1 ) { - // TODO: add metadata to configure this. - String nr = getVariable( Const.INTERNAL_VARIABLE_SLAVE_SERVER_NUMBER ); - if ( log.isDetailed() ) { - logDetailed( "Running on slave server #" + nr - + " : assuming that each slave reads a dedicated part of the same file(s)." ); - } - } - - // If no nullif field is supplied, take the default. - // String null_value = nullif; - // if (null_value == null) - // { - // // value=""; - // } - // String null_cmp = Const.rightPad(new StringBuilder(null_value), pol.length()); - - // calculate the file format type in advance so we can use a switch - data.fileFormatType = meta.getFileFormatTypeNr(); - - // calculate the file type in advance CSV or Fixed? - data.fileType = meta.getFileTypeNr(); - - // Handle the possibility of a variable substitution - data.separator = environmentSubstitute( meta.getSeparator() ); - data.enclosure = environmentSubstitute( meta.getEnclosure() ); - data.escapeCharacter = environmentSubstitute( meta.getEscapeCharacter() ); - - // Add additional fields - if ( !Const.isEmpty( meta.getShortFileNameField() ) ) { - data.addShortFilename = true; - } - if ( !Const.isEmpty( meta.getPathField() ) ) { - data.addPath = true; - } - if ( !Const.isEmpty( meta.getExtensionField() ) ) { - data.addExtension = true; - } - if ( !Const.isEmpty( meta.getSizeField() ) ) { - data.addSize = true; - } - if ( !Const.isEmpty( meta.isHiddenField() ) ) { - data.addIsHidden = true; - } - if ( !Const.isEmpty( meta.getLastModificationDateField() ) ) { - data.addLastModificationDate = true; - } - if ( !Const.isEmpty( meta.getUriField() ) ) { - data.addUri = true; - } - if ( !Const.isEmpty( meta.getRootUriField() ) ) { - data.addRootUri = true; - } - return true; - } - return false; - } - - private void initReplayFactory() { - Date replayDate = getTrans().getReplayDate(); - if ( replayDate == null ) { - data.filePlayList = FilePlayListAll.INSTANCE; - } else { - data.filePlayList = - new FilePlayListReplay( replayDate, meta.getLineNumberFilesDestinationDirectory(), meta - .getLineNumberFilesExtension(), meta.getErrorFilesDestinationDirectory(), meta - .getErrorLineFilesExtension(), meta.getEncoding() ); - } - } - - private void initErrorHandling() { - List dataErrorLineHandlers = new ArrayList( 2 ); - if ( meta.getLineNumberFilesDestinationDirectory() != null ) { - dataErrorLineHandlers - .add( new FileErrorHandlerContentLineNumber( getTrans().getCurrentDate(), environmentSubstitute( meta - .getLineNumberFilesDestinationDirectory() ), meta.getLineNumberFilesExtension(), meta.getEncoding(), this ) ); - } - if ( meta.getErrorFilesDestinationDirectory() != null ) { - dataErrorLineHandlers.add( new FileErrorHandlerMissingFiles( getTrans().getCurrentDate(), environmentSubstitute( - meta.getErrorFilesDestinationDirectory() ), meta.getErrorLineFilesExtension(), meta.getEncoding(), this ) ); - } - data.dataErrorLineHandler = new CompositeFileErrorHandler( dataErrorLineHandlers ); - } - - @Override - public void dispose( StepMetaInterface smi, StepDataInterface sdi ) { - meta = (TextFileInputMeta) smi; - data = (TextFileInputData) sdi; - - if ( data.file != null ) { - try { - data.file.close(); - data.file = null; - } catch ( Exception e ) { - log.logError( "Error closing file", e ); - } - } - if ( data.in != null ) { - BaseStep.closeQuietly( data.in ); - data.in = null; - } - super.dispose( smi, sdi ); - } - - public boolean isWaitingForData() { - return true; - } } diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputData.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputData.java index 219701dff173..c57eaa01f6fc 100644 --- a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputData.java +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputData.java @@ -22,59 +22,24 @@ package org.pentaho.di.trans.steps.textfileinput; -import java.io.InputStreamReader; -import java.util.Date; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import org.apache.commons.vfs2.FileObject; -import org.pentaho.di.core.RowSet; -import org.pentaho.di.core.compress.CompressionInputStream; -import org.pentaho.di.core.fileinput.FileInputList; import org.pentaho.di.core.playlist.FilePlayList; -import org.pentaho.di.core.row.RowMetaInterface; -import org.pentaho.di.trans.step.BaseStepData; -import org.pentaho.di.trans.step.StepDataInterface; -import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.steps.baseinput.BaseInputStepData; /** * @author Matt * @since 22-jan-2005 */ -public class TextFileInputData extends BaseStepData implements StepDataInterface { +public class TextFileInputData extends BaseInputStepData { public List lineBuffer; public Object[] previous_row; - public int nr_repeats; - public int nrLinesOnPage; - private FileInputList files; - - public HashMap passThruFields; - - public Object[] currentPassThruFieldsRow; - - public int nrPassThruFields; - - public boolean isLastFile; - - public String filename; - - public int lineInFile; - - public FileObject file; - - public int filenr; - - public CompressionInputStream in; - - public InputStreamReader isr; - public boolean doneReading; public int headerLinesRead; @@ -85,24 +50,16 @@ public class TextFileInputData extends BaseStepData implements StepDataInterface public boolean doneWithHeader; - public FileErrorHandler dataErrorLineHandler; - public FilePlayList filePlayList; public TextFileFilterProcessor filterProcessor; - public RowMetaInterface outputRowMeta; - public StringBuilder lineStringBuilder; public int fileFormatType; public int fileType; - public RowMetaInterface convertRowMeta; - - public RowSet rowSet; - /** * The separator (delimiter) */ @@ -112,53 +69,19 @@ public class TextFileInputData extends BaseStepData implements StepDataInterface public String escapeCharacter; - public boolean addShortFilename; - public boolean addExtension; - public boolean addPath; - public boolean addSize; - public boolean addIsHidden; - public boolean addLastModificationDate; - public boolean addUri; - public boolean addRootUri; - - public String shortFilename; - public String path; - public String extension; - public boolean hidden; - public Date lastModificationDateTime; - public String uriName; - public String rootUriName; - public long size; - public EncodingType encodingType; - - public Map rejectedFiles; + public TextFileInputData() { - super(); - // linked list is better, as usually .remove(0) is applied to this list lineBuffer = new LinkedList(); nr_repeats = 0; previous_row = null; - filenr = 0; nrLinesOnPage = 0; - in = null; - filterProcessor = null; lineStringBuilder = new StringBuilder( 256 ); - - rejectedFiles = new HashMap(); - } - - public FileInputList getFiles() { - return files; - } - - public void setFiles( FileInputList files ) { - this.files = files; } } diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputField.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputField.java index aa1fb4ea69a2..806c813f3aac 100644 --- a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputField.java +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputField.java @@ -39,7 +39,6 @@ * * @author Matt * @since 19-04-2004 - * */ public class TextFileInputField implements Cloneable, TextFileInputFieldInterface { private String name; diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java index c5df0f8befb5..e858c909cee4 100644 --- a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java @@ -1,4 +1,3 @@ -// CHECKSTYLE:FileLength:OFF /*! ****************************************************************************** * * Pentaho Data Integration @@ -23,14 +22,12 @@ package org.pentaho.di.trans.steps.textfileinput; -import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; -import com.google.common.annotations.VisibleForTesting; - import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.StringUtils; import org.apache.commons.vfs2.FileObject; import org.pentaho.di.core.CheckResult; import org.pentaho.di.core.CheckResultInterface; @@ -52,31 +49,22 @@ import org.pentaho.di.repository.ObjectId; import org.pentaho.di.repository.Repository; import org.pentaho.di.resource.ResourceDefinition; -import org.pentaho.di.resource.ResourceEntry; -import org.pentaho.di.resource.ResourceEntry.ResourceType; import org.pentaho.di.resource.ResourceNamingInterface; -import org.pentaho.di.resource.ResourceReference; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; -import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDataInterface; import org.pentaho.di.trans.step.StepInterface; import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMetaInjectionInterface; import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.baseinput.BaseInputStepMeta; import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; -public class TextFileInputMeta extends BaseStepMeta implements StepMetaInterface, InputFileMetaInterface { - private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! - - public static final String[] RequiredFilesDesc = new String[] { - BaseMessages.getString( PKG, "System.Combo.No" ), BaseMessages.getString( PKG, "System.Combo.Yes" ) }; - public static final String[] RequiredFilesCode = new String[] { "N", "Y" }; - - private static final String NO = "N"; +import com.google.common.annotations.VisibleForTesting; - private static final String YES = "Y"; +public class TextFileInputMeta extends BaseInputStepMeta implements StepMetaInterface { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! TODO: check i18n for base private static final String STRING_BASE64_PREFIX = "Base64: "; @@ -86,102 +74,105 @@ public class TextFileInputMeta extends BaseStepMeta implements StepMetaInterface public static final int FILE_TYPE_CSV = 0; public static final int FILE_TYPE_FIXED = 1; + + public Content content = new Content(); + + public static class Content implements Cloneable { - /** Array of filenames */ - private String[] fileName; + /** Type of file: CSV or fixed */ + public String fileType; - /** Wildcard or filemask (regular expression) */ - private String[] fileMask; + /** String used to separated field (;) */ + public String separator; - /** Wildcard or filemask to exclude (regular expression) */ - private String[] excludeFileMask; + /** String used to enclose separated fields (") */ + public String enclosure; - /** Array of boolean values as string, indicating if a file is required. */ - private String[] fileRequired; - /** Type of file: CSV or fixed */ - private String fileType; + /** Switch to allow breaks (CR/LF) in Enclosures */ + public boolean breakInEnclosureAllowed; + + /** Escape character used to escape the enclosure String (\) */ + public String escapeCharacter; - /** String used to separated field (;) */ - private String separator; - /** String used to enclose separated fields (") */ - private String enclosure; + /** Flag indicating that the file contains one header line that should be skipped. */ + public boolean header; - /** Escape character used to escape the enclosure String (\) */ - private String escapeCharacter; + /** The number of header lines, defaults to 1 */ + public int nrHeaderLines; - /** Switch to allow breaks (CR/LF) in Enclosures */ - private boolean breakInEnclosureAllowed; + /** Flag indicating that the file contains one footer line that should be skipped. */ + public boolean footer; - /** Flag indicating that the file contains one header line that should be skipped. */ - private boolean header; + /** The number of footer lines, defaults to 1 */ + public int nrFooterLines; - /** The number of header lines, defaults to 1 */ - private int nrHeaderLines; + /** Flag indicating that a single line is wrapped onto one or more lines in the text file. */ + public boolean lineWrapped; - /** Flag indicating that the file contains one footer line that should be skipped. */ - private boolean footer; + /** The number of times the line wrapped */ + public int nrWraps; + - /** The number of footer lines, defaults to 1 */ - private int nrFooterLines; + /** Flag indicating that the text-file has a paged layout. */ + public boolean layoutPaged; - /** Flag indicating that a single line is wrapped onto one or more lines in the text file. */ - private boolean lineWrapped; + /** The number of lines to read per page */ + public int nrLinesPerPage; + + /** The number of lines in the document header */ + public int nrLinesDocHeader; - /** The number of times the line wrapped */ - private int nrWraps; + /** Type of compression being used */ + public String fileCompression; - /** Flag indicating that the text-file has a paged layout. */ - private boolean layoutPaged; + /** Flag indicating that we should skip all empty lines */ + public boolean noEmptyLines; - /** The number of lines in the document header */ - private int nrLinesDocHeader; + /** Flag indicating that we should include the filename in the output */ + public boolean includeFilename; - /** The number of lines to read per page */ - private int nrLinesPerPage; + + /** The name of the field in the output containing the filename */ + public String filenameField; - /** Type of compression being used */ - private String fileCompression; + /** Flag indicating that a row number field should be included in the output */ + public boolean includeRowNumber; - /** Flag indicating that we should skip all empty lines */ - private boolean noEmptyLines; + /** The name of the field in the output containing the row number */ + public String rowNumberField; + + /** Flag indicating row number is per file */ + public boolean rowNumberByFile; - /** Flag indicating that we should include the filename in the output */ - private boolean includeFilename; + - /** The name of the field in the output containing the filename */ - private String filenameField; + /** The file format: DOS or UNIX or mixed */ + public String fileFormat; - /** Flag indicating that a row number field should be included in the output */ - private boolean includeRowNumber; + /** The encoding to use for reading: null or empty string means system default encoding */ + public String encoding; - /** Flag indicating row number is per file */ - private boolean rowNumberByFile; + /** The maximum number or lines to read */ + public long rowLimit; + + /** Indicate whether or not we want to date fields strictly according to the format or lenient */ + public boolean dateFormatLenient; + + /** Specifies the Locale of the Date format, null means the default */ + public Locale dateFormatLocale; + + } - /** The name of the field in the output containing the row number */ - private String rowNumberField; - /** The file format: DOS or UNIX or mixed */ - private String fileFormat; - /** The maximum number or lines to read */ - private long rowLimit; - /** The fields to import... */ - private TextFileInputField[] inputFields; - /** Array of boolean values as string, indicating if we need to fetch sub folders. */ - private String[] includeSubFolders; /** The filters to use... */ private TextFileFilter[] filter; - /** The encoding to use for reading: null or empty string means system default encoding */ - private String encoding; - - /** Ignore error : turn into warnings */ - private boolean errorIgnored; /** The name of the field that will contain the number of errors in the row */ private String errorCountField; @@ -192,370 +183,17 @@ public class TextFileInputMeta extends BaseStepMeta implements StepMetaInterface /** The name of the field that will contain the error texts, separated by CR */ private String errorTextField; - /** The directory that will contain warning files */ - private String warningFilesDestinationDirectory; - - /** The extension of warning files */ - private String warningFilesExtension; - - /** The directory that will contain error files */ - private String errorFilesDestinationDirectory; - - /** The extension of error files */ - private String errorFilesExtension; - - /** The directory that will contain line number files */ - private String lineNumberFilesDestinationDirectory; - - /** The extension of line number files */ - private String lineNumberFilesExtension; - - /** Indicate whether or not we want to date fields strictly according to the format or lenient */ - private boolean dateFormatLenient; - - /** Specifies the Locale of the Date format, null means the default */ - private Locale dateFormatLocale; - /** If error line are skipped, you can replay without introducing doubles. */ private boolean errorLineSkipped; - /** Are we accepting filenames in input rows? */ - private boolean acceptingFilenames; - - /** If receiving input rows, should we pass through existing fields? */ - private boolean passingThruFields; - - /** The field in which the filename is placed */ - private String acceptingField; - - /** The stepname to accept filenames from */ - private String acceptingStepName; - /** The step to accept filenames from */ private StepMeta acceptingStep; - /** The add filenames to result filenames flag */ - private boolean isaddresult; - - /** Additional fields **/ - private String shortFileFieldName; - private String pathFieldName; - private String hiddenFieldName; - private String lastModificationTimeFieldName; - private String uriNameFieldName; - private String rootUriNameFieldName; - private String extensionFieldName; - private String sizeFieldName; - - private boolean skipBadFiles; - private String fileErrorField; - private String fileErrorMessageField; - - /** - * @return Returns the shortFileFieldName. - */ - public String getShortFileNameField() { - return shortFileFieldName; - } - - /** - * @param field - * The shortFileFieldName to set. - */ - public void setShortFileNameField( String field ) { - shortFileFieldName = field; - } - - /** - * @return Returns the pathFieldName. - */ - public String getPathField() { - return pathFieldName; - } - - /** - * @param field - * The pathFieldName to set. - */ - public void setPathField( String field ) { - this.pathFieldName = field; - } - - /** - * @return Returns the hiddenFieldName. - */ - public String isHiddenField() { - return hiddenFieldName; - } - - /** - * @param field - * The hiddenFieldName to set. - */ - public void setIsHiddenField( String field ) { - hiddenFieldName = field; - } - - /** - * @return Returns the lastModificationTimeFieldName. - */ - public String getLastModificationDateField() { - return lastModificationTimeFieldName; - } - - /** - * @param field - * The lastModificationTimeFieldName to set. - */ - public void setLastModificationDateField( String field ) { - lastModificationTimeFieldName = field; - } - - /** - * @return Returns the uriNameFieldName. - */ - public String getUriField() { - return uriNameFieldName; - } - - /** - * @param field - * The uriNameFieldName to set. - */ - public void setUriField( String field ) { - uriNameFieldName = field; - } - - /** - * @return Returns the uriNameFieldName. - */ - public String getRootUriField() { - return rootUriNameFieldName; - } - - /** - * @param field - * The rootUriNameFieldName to set. - */ - public void setRootUriField( String field ) { - rootUriNameFieldName = field; - } - - /** - * @return Returns the extensionFieldName. - */ - public String getExtensionField() { - return extensionFieldName; - } - - /** - * @param field - * The extensionFieldName to set. - */ - public void setExtensionField( String field ) { - extensionFieldName = field; - } - - /** - * @return Returns the sizeFieldName. - */ - public String getSizeField() { - return sizeFieldName; - } - - /** - * @param field - * The sizeFieldName to set. - */ - public void setSizeField( String field ) { - sizeFieldName = field; - } - - /** - * - * @return If should continue processing after failing to open a file - */ - public boolean isSkipBadFiles() { - return skipBadFiles; - } - - /** - * - * @param value - * If should continue processing after failing to open a file - */ - public void setSkipBadFiles( boolean value ) { - skipBadFiles = value; - } - - public String getFileErrorField() { - return fileErrorField; - } - - public void setFileErrorField( String field ) { - fileErrorField = field; - } - - public String getFileErrorMessageField() { - return fileErrorMessageField; - } - - public void setFileErrorMessageField( String field ) { - fileErrorMessageField = field; - } - - /** - * @return Returns the encoding. - */ - public String getEncoding() { - return encoding; - } - - /** - * @param encoding - * The encoding to set. - */ - public void setEncoding( String encoding ) { - this.encoding = encoding; - } - - public TextFileInputMeta() { - super(); // allocate BaseStepMeta - } - - /** - * @return Returns the input fields. - */ - public TextFileInputField[] getInputFields() { - return inputFields; - } - - /** - * @param inputFields - * The input fields to set. - */ - public void setInputFields( TextFileInputField[] inputFields ) { - this.inputFields = inputFields; - } - - /** - * @return Returns the enclosure. - */ - public String getEnclosure() { - return enclosure; - } - - /** - * @param enclosure - * The enclosure to set. - */ - public void setEnclosure( String enclosure ) { - this.enclosure = enclosure; - } - - /** - * @return Returns the breakInEnclosureAllowed. - */ - public boolean isBreakInEnclosureAllowed() { - return breakInEnclosureAllowed; - } - - /** - * @param breakInEnclosureAllowed - * The breakInEnclosureAllowed to set. - */ - public void setBreakInEnclosureAllowed( boolean breakInEnclosureAllowed ) { - this.breakInEnclosureAllowed = breakInEnclosureAllowed; - } - - /** - * @return Returns the excludeFileMask. - */ - public String[] getExludeFileMask() { - return excludeFileMask; - } - - /** - * @param excludeFileMask - * The excludeFileMask to set. - */ - public void setExcludeFileMask( String[] excludeFileMask ) { - this.excludeFileMask = excludeFileMask; - } - - /** - * @return Returns the fileFormat. - */ - public String getFileFormat() { - return fileFormat; - } - - /** - * @param fileFormat - * The fileFormat to set. - */ - public void setFileFormat( String fileFormat ) { - this.fileFormat = fileFormat; - } - - /** - * @return Returns the fileMask. - */ - public String[] getFileMask() { - return fileMask; - } - - /** - * @return Returns the fileRequired. - */ - public String[] getFileRequired() { - return fileRequired; - } - - /** - * @param fileMask - * The fileMask to set. - */ - public void setFileMask( String[] fileMask ) { - this.fileMask = fileMask; - } - - /** - * @param fileRequired - * The fileRequired to set. - */ - public void setFileRequired( String[] fileRequiredin ) { - for ( int i = 0; i < fileRequiredin.length; i++ ) { - this.fileRequired[i] = getRequiredFilesCode( fileRequiredin[i] ); - } - } - - public String[] getIncludeSubFolders() { - return includeSubFolders; - } - - public void setIncludeSubFolders( String[] includeSubFoldersin ) { - for ( int i = 0; i < includeSubFoldersin.length; i++ ) { - this.includeSubFolders[i] = getRequiredFilesCode( includeSubFoldersin[i] ); - } - } - - public String getRequiredFilesCode( String tt ) { - if ( tt == null ) { - return RequiredFilesCode[0]; - } - if ( tt.equals( RequiredFilesDesc[1] ) ) { - return RequiredFilesCode[1]; - } else { - return RequiredFilesCode[0]; - } - } - /** * @return Returns the fileName. */ public String[] getFileName() { - return fileName; + return inputFiles.fileName; } /** @@ -563,37 +201,7 @@ public String[] getFileName() { * The fileName to set. */ public void setFileName( String[] fileName ) { - this.fileName = fileName; - } - - /** - * @return Returns the filenameField. - */ - public String getFilenameField() { - return filenameField; - } - - /** - * @param filenameField - * The filenameField to set. - */ - public void setFilenameField( String filenameField ) { - this.filenameField = filenameField; - } - - /** - * @return Returns the fileType. - */ - public String getFileType() { - return fileType; - } - - /** - * @param fileType - * The fileType to set. - */ - public void setFileType( String fileType ) { - this.fileType = fileType; + inputFiles.fileName = fileName; } /** @@ -611,194 +219,42 @@ public void setFilter( TextFileFilter[] filter ) { this.filter = filter; } - /** - * @return Returns the footer. - */ - public boolean hasFooter() { - return footer; - } - - /** - * @param footer - * The footer to set. - */ - public void setFooter( boolean footer ) { - this.footer = footer; - } - - /** - * @return Returns the header. - */ - public boolean hasHeader() { - return header; - } - - /** - * @param header - * The header to set. - */ - public void setHeader( boolean header ) { - this.header = header; - } - - /** - * @return Returns the includeFilename. - */ - public boolean includeFilename() { - return includeFilename; - } - - /** - * @param includeFilename - * The includeFilename to set. - */ - public void setIncludeFilename( boolean includeFilename ) { - this.includeFilename = includeFilename; - } - - /** - * @return Returns the includeRowNumber. - */ - public boolean includeRowNumber() { - return includeRowNumber; - } - - /** - * @param includeRowNumber - * The includeRowNumber to set. - */ - public void setIncludeRowNumber( boolean includeRowNumber ) { - this.includeRowNumber = includeRowNumber; - } - - /** - * true if row number reset for each file - * - * @return rowNumberByFile - */ - public boolean isRowNumberByFile() { - return rowNumberByFile; - } - - /** - * @param rowNumberByFile - * True if row number field is reset for each file - */ - public void setRowNumberByFile( boolean rowNumberByFile ) { - this.rowNumberByFile = rowNumberByFile; - } - - /** - * @return Returns the noEmptyLines. - */ - public boolean noEmptyLines() { - return noEmptyLines; - } - - /** - * @param noEmptyLines - * The noEmptyLines to set. - */ - public void setNoEmptyLines( boolean noEmptyLines ) { - this.noEmptyLines = noEmptyLines; - } - - /** - * @return Returns the rowLimit. - */ - public long getRowLimit() { - return rowLimit; - } - - /** - * @param rowLimit - * The rowLimit to set. - */ - public void setRowLimit( long rowLimit ) { - this.rowLimit = rowLimit; - } - - /** - * @return Returns the rowNumberField. - */ - public String getRowNumberField() { - return rowNumberField; - } - - /** - * @param rowNumberField - * The rowNumberField to set. - */ - public void setRowNumberField( String rowNumberField ) { - this.rowNumberField = rowNumberField; - } - - /** - * @return Returns the separator. - */ - public String getSeparator() { - return separator; - } - - /** - * @param separator - * The separator to set. - */ - public void setSeparator( String separator ) { - this.separator = separator; - } - - /** - * @return Returns the type of compression used - */ - public String getFileCompression() { - return fileCompression; - } - - /** - * @param fileCompression - * Sets the compression type - */ - public void setFileCompression( String fileCompression ) { - this.fileCompression = fileCompression; - } - public void loadXML( Node stepnode, List databases, IMetaStore metaStore ) throws KettleXMLException { try { - acceptingFilenames = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "accept_filenames" ) ); - passingThruFields = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "passing_through_fields" ) ); - acceptingField = XMLHandler.getTagValue( stepnode, "accept_field" ); - acceptingStepName = XMLHandler.getTagValue( stepnode, "accept_stepname" ); - - separator = XMLHandler.getTagValue( stepnode, "separator" ); - enclosure = XMLHandler.getTagValue( stepnode, "enclosure" ); - breakInEnclosureAllowed = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "enclosure_breaks" ) ); - escapeCharacter = XMLHandler.getTagValue( stepnode, "escapechar" ); - header = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) ); - nrHeaderLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_headerlines" ), 1 ); - footer = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "footer" ) ); - nrFooterLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_footerlines" ), 1 ); - lineWrapped = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "line_wrapped" ) ); - nrWraps = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_wraps" ), 1 ); - layoutPaged = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "layout_paged" ) ); - nrLinesPerPage = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_per_page" ), 1 ); - nrLinesDocHeader = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_doc_header" ), 1 ); + inputFiles.acceptingFilenames = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "accept_filenames" ) ); + inputFiles.passingThruFields = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "passing_through_fields" ) ); + inputFiles.acceptingField = XMLHandler.getTagValue( stepnode, "accept_field" ); + inputFiles.acceptingStepName = XMLHandler.getTagValue( stepnode, "accept_stepname" ); + + content.separator = XMLHandler.getTagValue( stepnode, "separator" ); + content.enclosure = XMLHandler.getTagValue( stepnode, "enclosure" ); + content.breakInEnclosureAllowed = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "enclosure_breaks" ) ); + content.escapeCharacter = XMLHandler.getTagValue( stepnode, "escapechar" ); + content.header = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) ); + content.nrHeaderLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_headerlines" ), 1 ); + content.footer = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "footer" ) ); + content.nrFooterLines = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_footerlines" ), 1 ); + content.lineWrapped = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "line_wrapped" ) ); + content.nrWraps = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_wraps" ), 1 ); + content.layoutPaged = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "layout_paged" ) ); + content.nrLinesPerPage = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_per_page" ), 1 ); + content.nrLinesDocHeader = Const.toInt( XMLHandler.getTagValue( stepnode, "nr_lines_doc_header" ), 1 ); String addToResult = XMLHandler.getTagValue( stepnode, "add_to_result_filenames" ); if ( Const.isEmpty( addToResult ) ) { - isaddresult = true; + inputFiles.isaddresult = true; } else { - isaddresult = "Y".equalsIgnoreCase( addToResult ); + inputFiles.isaddresult = "Y".equalsIgnoreCase( addToResult ); } String nempty = XMLHandler.getTagValue( stepnode, "noempty" ); - noEmptyLines = YES.equalsIgnoreCase( nempty ) || nempty == null; - includeFilename = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include" ) ); - filenameField = XMLHandler.getTagValue( stepnode, "include_field" ); - includeRowNumber = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownum" ) ); - rowNumberByFile = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownumByFile" ) ); - rowNumberField = XMLHandler.getTagValue( stepnode, "rownum_field" ); - fileFormat = XMLHandler.getTagValue( stepnode, "format" ); - encoding = XMLHandler.getTagValue( stepnode, "encoding" ); + content.noEmptyLines = YES.equalsIgnoreCase( nempty ) || nempty == null; + content.includeFilename = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include" ) ); + content.filenameField = XMLHandler.getTagValue( stepnode, "include_field" ); + content.includeRowNumber = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownum" ) ); + content.rowNumberByFile = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownumByFile" ) ); + content.rowNumberField = XMLHandler.getTagValue( stepnode, "rownum_field" ); + content.fileFormat = XMLHandler.getTagValue( stepnode, "format" ); + content.encoding = XMLHandler.getTagValue( stepnode, "encoding" ); Node filenode = XMLHandler.getSubNode( stepnode, "file" ); Node fields = XMLHandler.getSubNode( stepnode, "fields" ); @@ -815,19 +271,19 @@ public void loadXML( Node stepnode, List databases, IMetaStore met Node excludefilemasknode = XMLHandler.getSubNodeByNr( filenode, "exclude_filemask", i ); Node fileRequirednode = XMLHandler.getSubNodeByNr( filenode, "file_required", i ); Node includeSubFoldersnode = XMLHandler.getSubNodeByNr( filenode, "include_subfolders", i ); - fileName[i] = loadSource( filenode, filenamenode, i ); - fileMask[i] = XMLHandler.getNodeValue( filemasknode ); - excludeFileMask[i] = XMLHandler.getNodeValue( excludefilemasknode ); - fileRequired[i] = XMLHandler.getNodeValue( fileRequirednode ); - includeSubFolders[i] = XMLHandler.getNodeValue( includeSubFoldersnode ); + inputFiles.fileName[i] = loadSource( filenode, filenamenode, i ); + inputFiles.fileMask[i] = XMLHandler.getNodeValue( filemasknode ); + inputFiles. excludeFileMask[i] = XMLHandler.getNodeValue( excludefilemasknode ); + inputFiles. fileRequired[i] = XMLHandler.getNodeValue( fileRequirednode ); + inputFiles.includeSubFolders[i] = XMLHandler.getNodeValue( includeSubFoldersnode ); } - fileType = XMLHandler.getTagValue( stepnode, "file", "type" ); - fileCompression = XMLHandler.getTagValue( stepnode, "file", "compression" ); - if ( fileCompression == null ) { - fileCompression = "None"; + content.fileType = XMLHandler.getTagValue( stepnode, "file", "type" ); + content.fileCompression = XMLHandler.getTagValue( stepnode, "file", "compression" ); + if ( content.fileCompression == null ) { + content.fileCompression = "None"; if ( YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "zipped" ) ) ) { - fileCompression = "Zip"; + content.fileCompression = "Zip"; } } @@ -878,45 +334,45 @@ public void loadXML( Node stepnode, List databases, IMetaStore met field.setTrimType( ValueMeta.getTrimTypeByCode( XMLHandler.getTagValue( fnode, "trim_type" ) ) ); field.setRepeated( YES.equalsIgnoreCase( XMLHandler.getTagValue( fnode, "repeat" ) ) ); - inputFields[i] = field; + inputFiles.inputFields[i] = field; } // Is there a limit on the number of rows we process? - rowLimit = Const.toLong( XMLHandler.getTagValue( stepnode, "limit" ), 0L ); + content.rowLimit = Const.toLong( XMLHandler.getTagValue( stepnode, "limit" ), 0L ); - errorIgnored = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_ignored" ) ); - skipBadFiles = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "skip_bad_files" ) ); - fileErrorField = XMLHandler.getTagValue( stepnode, "file_error_field" ); - fileErrorMessageField = XMLHandler.getTagValue( stepnode, "file_error_message_field" ); + errorHandling.errorIgnored = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_ignored" ) ); + errorHandling.skipBadFiles = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "skip_bad_files" ) ); + errorHandling.fileErrorField = XMLHandler.getTagValue( stepnode, "file_error_field" ); + errorHandling.fileErrorMessageField = XMLHandler.getTagValue( stepnode, "file_error_message_field" ); errorLineSkipped = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_line_skipped" ) ); errorCountField = XMLHandler.getTagValue( stepnode, "error_count_field" ); errorFieldsField = XMLHandler.getTagValue( stepnode, "error_fields_field" ); errorTextField = XMLHandler.getTagValue( stepnode, "error_text_field" ); - warningFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "bad_line_files_destination_directory" ); - warningFilesExtension = XMLHandler.getTagValue( stepnode, "bad_line_files_extension" ); - errorFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "error_line_files_destination_directory" ); - errorFilesExtension = XMLHandler.getTagValue( stepnode, "error_line_files_extension" ); - lineNumberFilesDestinationDirectory = + errorHandling. warningFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "bad_line_files_destination_directory" ); + errorHandling.warningFilesExtension = XMLHandler.getTagValue( stepnode, "bad_line_files_extension" ); + errorHandling.errorFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "error_line_files_destination_directory" ); + errorHandling.errorFilesExtension = XMLHandler.getTagValue( stepnode, "error_line_files_extension" ); + errorHandling.lineNumberFilesDestinationDirectory = XMLHandler.getTagValue( stepnode, "line_number_files_destination_directory" ); - lineNumberFilesExtension = XMLHandler.getTagValue( stepnode, "line_number_files_extension" ); + errorHandling.lineNumberFilesExtension = XMLHandler.getTagValue( stepnode, "line_number_files_extension" ); // Backward compatible - dateFormatLenient = !NO.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "date_format_lenient" ) ); + content.dateFormatLenient = !NO.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "date_format_lenient" ) ); String dateLocale = XMLHandler.getTagValue( stepnode, "date_format_locale" ); if ( dateLocale != null ) { - dateFormatLocale = EnvUtil.createLocale( dateLocale ); + content.dateFormatLocale = EnvUtil.createLocale( dateLocale ); } else { - dateFormatLocale = Locale.getDefault(); + content.dateFormatLocale = Locale.getDefault(); } - shortFileFieldName = XMLHandler.getTagValue( stepnode, "shortFileFieldName" ); - pathFieldName = XMLHandler.getTagValue( stepnode, "pathFieldName" ); - hiddenFieldName = XMLHandler.getTagValue( stepnode, "hiddenFieldName" ); - lastModificationTimeFieldName = XMLHandler.getTagValue( stepnode, "lastModificationTimeFieldName" ); - uriNameFieldName = XMLHandler.getTagValue( stepnode, "uriNameFieldName" ); - rootUriNameFieldName = XMLHandler.getTagValue( stepnode, "rootUriNameFieldName" ); - extensionFieldName = XMLHandler.getTagValue( stepnode, "extensionFieldName" ); - sizeFieldName = XMLHandler.getTagValue( stepnode, "sizeFieldName" ); + additionalOutputFields.shortFilenameField = XMLHandler.getTagValue( stepnode, "shortFileFieldName" ); + additionalOutputFields.pathField = XMLHandler.getTagValue( stepnode, "pathFieldName" ); + additionalOutputFields.hiddenField = XMLHandler.getTagValue( stepnode, "hiddenFieldName" ); + additionalOutputFields.lastModificationField = XMLHandler.getTagValue( stepnode, "lastModificationTimeFieldName" ); + additionalOutputFields.uriField = XMLHandler.getTagValue( stepnode, "uriNameFieldName" ); + additionalOutputFields.rootUriField = XMLHandler.getTagValue( stepnode, "rootUriNameFieldName" ); + additionalOutputFields.extensionField = XMLHandler.getTagValue( stepnode, "extensionFieldName" ); + additionalOutputFields.sizeField = XMLHandler.getTagValue( stepnode, "sizeFieldName" ); } catch ( Exception e ) { throw new KettleXMLException( "Unable to load step info from XML", e ); } @@ -925,30 +381,30 @@ public void loadXML( Node stepnode, List databases, IMetaStore met public Object clone() { TextFileInputMeta retval = (TextFileInputMeta) super.clone(); - int nrfiles = fileName.length; - int nrfields = inputFields.length; + int nrfiles = inputFiles.fileName.length; + int nrfields = inputFiles.inputFields.length; int nrfilters = filter.length; retval.allocate( nrfiles, nrfields, nrfilters ); for ( int i = 0; i < nrfiles; i++ ) { - retval.fileName[i] = fileName[i]; - retval.fileMask[i] = fileMask[i]; - retval.excludeFileMask[i] = excludeFileMask[i]; - retval.fileRequired[i] = fileRequired[i]; - retval.includeSubFolders[i] = includeSubFolders[i]; + retval.inputFiles.fileName[i] = inputFiles.fileName[i]; + retval.inputFiles.fileMask[i] = inputFiles.fileMask[i]; + retval.inputFiles.excludeFileMask[i] = inputFiles.excludeFileMask[i]; + retval.inputFiles.fileRequired[i] = inputFiles.fileRequired[i]; + retval.inputFiles.includeSubFolders[i] = inputFiles.includeSubFolders[i]; } for ( int i = 0; i < nrfields; i++ ) { - retval.inputFields[i] = (TextFileInputField) inputFields[i].clone(); + retval.inputFiles.inputFields[i] = (TextFileInputField) inputFiles.inputFields[i].clone(); } for ( int i = 0; i < nrfilters; i++ ) { retval.filter[i] = (TextFileFilter) filter[i].clone(); } - retval.dateFormatLocale = (Locale) dateFormatLocale.clone(); - retval.fileCompression = fileCompression; + retval.content.dateFormatLocale = (Locale) content.dateFormatLocale.clone(); + retval.content.fileCompression = content.fileCompression; return retval; } @@ -956,60 +412,62 @@ public Object clone() { public void allocate( int nrfiles, int nrfields, int nrfilters ) { allocateFiles( nrfiles ); - inputFields = new TextFileInputField[nrfields]; + inputFiles.inputFields = new TextFileInputField[nrfields]; filter = new TextFileFilter[nrfilters]; } public void allocateFiles( int nrFiles ) { - fileName = new String[nrFiles]; - fileMask = new String[nrFiles]; - excludeFileMask = new String[nrFiles]; - fileRequired = new String[nrFiles]; - includeSubFolders = new String[nrFiles]; + inputFiles. fileName = new String[nrFiles]; + inputFiles. fileMask = new String[nrFiles]; + inputFiles. excludeFileMask = new String[nrFiles]; + inputFiles. fileRequired = new String[nrFiles]; + inputFiles. includeSubFolders = new String[nrFiles]; } public void setDefault() { - shortFileFieldName = null; - pathFieldName = null; - hiddenFieldName = null; - lastModificationTimeFieldName = null; - uriNameFieldName = null; - rootUriNameFieldName = null; - extensionFieldName = null; - sizeFieldName = null; - - isaddresult = true; - separator = ";"; - enclosure = "\""; - breakInEnclosureAllowed = false; - header = true; - nrHeaderLines = 1; - footer = false; - nrFooterLines = 1; - lineWrapped = false; - nrWraps = 1; - layoutPaged = false; - nrLinesPerPage = 80; - nrLinesDocHeader = 0; - fileCompression = "None"; - noEmptyLines = true; - fileFormat = "DOS"; - fileType = "CSV"; - includeFilename = false; - filenameField = ""; - includeRowNumber = false; - rowNumberField = ""; - errorIgnored = false; - skipBadFiles = false; + additionalOutputFields.shortFilenameField = null; + additionalOutputFields.pathField = null; + additionalOutputFields.hiddenField = null; + additionalOutputFields.lastModificationField = null; + additionalOutputFields.uriField = null; + additionalOutputFields.rootUriField = null; + additionalOutputFields.extensionField = null; + additionalOutputFields.sizeField = null; + + inputFiles.isaddresult = true; + + content.separator = ";"; + content.enclosure = "\""; + content.breakInEnclosureAllowed = false; + content.header = true; + content.nrHeaderLines = 1; + content.footer = false; + content.nrFooterLines = 1; + content.lineWrapped = false; + content.nrWraps = 1; + content.layoutPaged = false; + content.nrLinesPerPage = 80; + content.nrLinesDocHeader = 0; + content.fileCompression = "None"; + content.noEmptyLines = true; + content.fileFormat = "DOS"; + content.fileType = "CSV"; + content.includeFilename = false; + content.filenameField = ""; + content.includeRowNumber = false; + content.rowNumberField = ""; + content.dateFormatLenient = true; + content.rowNumberByFile = false; + + errorHandling.errorIgnored = false; + errorHandling.skipBadFiles = false; errorLineSkipped = false; - warningFilesDestinationDirectory = null; - warningFilesExtension = "warning"; - errorFilesDestinationDirectory = null; - errorFilesExtension = "error"; - lineNumberFilesDestinationDirectory = null; - lineNumberFilesExtension = "line"; - dateFormatLenient = true; - rowNumberByFile = false; + errorHandling. warningFilesDestinationDirectory = null; + errorHandling. warningFilesExtension = "warning"; + errorHandling. errorFilesDestinationDirectory = null; + errorHandling. errorFilesExtension = "error"; + errorHandling. lineNumberFilesDestinationDirectory = null; + errorHandling. lineNumberFilesExtension = "line"; int nrfiles = 0; int nrfields = 0; @@ -1018,25 +476,25 @@ public void setDefault() { allocate( nrfiles, nrfields, nrfilters ); for ( int i = 0; i < nrfiles; i++ ) { - fileName[i] = "filename" + ( i + 1 ); - fileMask[i] = ""; - excludeFileMask[i] = ""; - fileRequired[i] = NO; - includeSubFolders[i] = NO; + inputFiles. fileName[i] = "filename" + ( i + 1 ); + inputFiles. fileMask[i] = ""; + inputFiles. excludeFileMask[i] = ""; + inputFiles. fileRequired[i] = NO; + inputFiles. includeSubFolders[i] = NO; } for ( int i = 0; i < nrfields; i++ ) { - inputFields[i] = new TextFileInputField( "field" + ( i + 1 ), 1, -1 ); + inputFiles.inputFields[i] = new TextFileInputField( "field" + ( i + 1 ), 1, -1 ); } - dateFormatLocale = Locale.getDefault(); + content.dateFormatLocale = Locale.getDefault(); - rowLimit = 0L; + content.rowLimit = 0L; } public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { - if ( !isPassingThruFields() ) { + if ( !inputFiles.passingThruFields ) { // all incoming fields are not transmitted ! row.clear(); } else { @@ -1051,8 +509,8 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf } } - for ( int i = 0; i < inputFields.length; i++ ) { - TextFileInputField field = inputFields[i]; + for ( int i = 0; i < inputFiles.inputFields.length; i++ ) { + TextFileInputField field = inputFiles.inputFields[i]; int type = field.getType(); if ( type == ValueMetaInterface.TYPE_NONE ) { @@ -1068,8 +526,8 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf v.setDecimalSymbol( field.getDecimalSymbol() ); v.setGroupingSymbol( field.getGroupSymbol() ); v.setCurrencySymbol( field.getCurrencySymbol() ); - v.setDateFormatLenient( dateFormatLenient ); - v.setDateFormatLocale( dateFormatLocale ); + v.setDateFormatLenient( content.dateFormatLenient ); + v.setDateFormatLocale( content.dateFormatLocale ); v.setTrimType( field.getTrimType() ); row.addValueMeta( v ); @@ -1077,7 +535,7 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf throw new KettleStepException( e ); } } - if ( errorIgnored ) { + if (errorHandling. errorIgnored ) { if ( errorCountField != null && errorCountField.length() > 0 ) { ValueMetaInterface v = new ValueMeta( errorCountField, ValueMetaInterface.TYPE_INTEGER ); v.setLength( ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0 ); @@ -1095,14 +553,14 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf row.addValueMeta( v ); } } - if ( includeFilename ) { - ValueMetaInterface v = new ValueMeta( filenameField, ValueMetaInterface.TYPE_STRING ); + if ( content.includeFilename ) { + ValueMetaInterface v = new ValueMeta( content.filenameField, ValueMetaInterface.TYPE_STRING ); v.setLength( 100 ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( includeRowNumber ) { - ValueMetaInterface v = new ValueMeta( rowNumberField, ValueMetaInterface.TYPE_INTEGER ); + if ( content.includeRowNumber ) { + ValueMetaInterface v = new ValueMeta( content.rowNumberField, ValueMetaInterface.TYPE_INTEGER ); v.setLength( ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0 ); v.setOrigin( name ); row.addValueMeta( v ); @@ -1110,58 +568,58 @@ public void getFields( RowMetaInterface row, String name, RowMetaInterface[] inf // Add additional fields - if ( getShortFileNameField() != null && getShortFileNameField().length() > 0 ) { + if ( StringUtils.isNotBlank( additionalOutputFields.shortFilenameField ) ) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( getShortFileNameField() ), ValueMetaInterface.TYPE_STRING ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.shortFilenameField ), ValueMetaInterface.TYPE_STRING ); v.setLength( 100, -1 ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getExtensionField() != null && getExtensionField().length() > 0 ) { + if ( StringUtils.isNotBlank( additionalOutputFields.extensionField ) ) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( getExtensionField() ), ValueMetaInterface.TYPE_STRING ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.extensionField ), ValueMetaInterface.TYPE_STRING ); v.setLength( 100, -1 ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getPathField() != null && getPathField().length() > 0 ) { + if ( StringUtils.isNotBlank(additionalOutputFields.pathField ) ) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( getPathField() ), ValueMetaInterface.TYPE_STRING ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.pathField ), ValueMetaInterface.TYPE_STRING ); v.setLength( 100, -1 ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getSizeField() != null && getSizeField().length() > 0 ) { + if ( StringUtils.isNotBlank( additionalOutputFields.sizeField ) ) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( getSizeField() ), ValueMetaInterface.TYPE_INTEGER ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.sizeField ), ValueMetaInterface.TYPE_INTEGER ); v.setOrigin( name ); v.setLength( 9 ); row.addValueMeta( v ); } - if ( isHiddenField() != null && isHiddenField().length() > 0 ) { + if ( StringUtils.isNotBlank( additionalOutputFields.hiddenField ) ) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( isHiddenField() ), ValueMetaInterface.TYPE_BOOLEAN ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.hiddenField ), ValueMetaInterface.TYPE_BOOLEAN ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getLastModificationDateField() != null && getLastModificationDateField().length() > 0 ) { + if ( StringUtils.isNotBlank(additionalOutputFields.lastModificationField) ) { ValueMetaInterface v = new ValueMeta( - space.environmentSubstitute( getLastModificationDateField() ), ValueMetaInterface.TYPE_DATE ); + space.environmentSubstitute( additionalOutputFields.lastModificationField ), ValueMetaInterface.TYPE_DATE ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getUriField() != null && getUriField().length() > 0 ) { + if ( StringUtils.isNotBlank( additionalOutputFields.uriField)) { ValueMetaInterface v = - new ValueMeta( space.environmentSubstitute( getUriField() ), ValueMetaInterface.TYPE_STRING ); + new ValueMeta( space.environmentSubstitute( additionalOutputFields.uriField ), ValueMetaInterface.TYPE_STRING ); v.setLength( 100, -1 ); v.setOrigin( name ); row.addValueMeta( v ); } - if ( getRootUriField() != null && getRootUriField().length() > 0 ) { - ValueMetaInterface v = new ValueMeta( getRootUriField(), ValueMetaInterface.TYPE_STRING ); + if ( StringUtils.isNotBlank(additionalOutputFields.rootUriField ) ) { + ValueMetaInterface v = new ValueMeta( additionalOutputFields.rootUriField, ValueMetaInterface.TYPE_STRING ); v.setLength( 100, -1 ); v.setOrigin( name ); row.addValueMeta( v ); @@ -1179,46 +637,46 @@ public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterf public String getXML() { StringBuffer retval = new StringBuffer( 1500 ); - retval.append( " " ).append( XMLHandler.addTagValue( "accept_filenames", acceptingFilenames ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "passing_through_fields", passingThruFields ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "accept_field", acceptingField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "accept_filenames", inputFiles.acceptingFilenames ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "passing_through_fields", inputFiles.passingThruFields ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "accept_field", inputFiles.acceptingField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "accept_stepname", ( acceptingStep != null ? acceptingStep.getName() : "" ) ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "separator", separator ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "enclosure", enclosure ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "enclosure_breaks", breakInEnclosureAllowed ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "escapechar", escapeCharacter ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "header", header ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "nr_headerlines", nrHeaderLines ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "footer", footer ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "nr_footerlines", nrFooterLines ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "line_wrapped", lineWrapped ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "nr_wraps", nrWraps ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "layout_paged", layoutPaged ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_per_page", nrLinesPerPage ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_doc_header", nrLinesDocHeader ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "noempty", noEmptyLines ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "include", includeFilename ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "include_field", filenameField ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "rownum", includeRowNumber ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "rownumByFile", rowNumberByFile ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "rownum_field", rowNumberField ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "format", fileFormat ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "encoding", encoding ) ); - retval.append( " " + XMLHandler.addTagValue( "add_to_result_filenames", isaddresult ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "separator", content.separator ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "enclosure", content.enclosure ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "enclosure_breaks", content.breakInEnclosureAllowed ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "escapechar", content.escapeCharacter ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "header", content.header ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_headerlines", content.nrHeaderLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "footer", content.footer ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_footerlines", content.nrFooterLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "line_wrapped", content.lineWrapped ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_wraps", content.nrWraps ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "layout_paged", content.layoutPaged ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_per_page", content.nrLinesPerPage ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "nr_lines_doc_header", content.nrLinesDocHeader ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "noempty", content.noEmptyLines ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include", content.includeFilename ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include_field", content.filenameField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownum", content.includeRowNumber ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownumByFile", content.rowNumberByFile ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rownum_field", content.rowNumberField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "format", content.fileFormat ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "encoding", content.encoding ) ); + retval.append( " " + XMLHandler.addTagValue( "add_to_result_filenames", inputFiles.isaddresult ) ); retval.append( " " ).append( Const.CR ); - for ( int i = 0; i < fileName.length; i++ ) { - saveSource( retval, fileName[i] ); - retval.append( " " ).append( XMLHandler.addTagValue( "filemask", fileMask[i] ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "exclude_filemask", excludeFileMask[i] ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "file_required", fileRequired[i] ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", includeSubFolders[i] ) ); + for ( int i = 0; i < inputFiles.fileName.length; i++ ) { + saveSource( retval, inputFiles.fileName[i] ); + retval.append( " " ).append( XMLHandler.addTagValue( "filemask", inputFiles.fileMask[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "exclude_filemask", inputFiles.excludeFileMask[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_required", inputFiles.fileRequired[i] ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", inputFiles.includeSubFolders[i] ) ); } - retval.append( " " ).append( XMLHandler.addTagValue( "type", fileType ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "type", content.fileType ) ); retval.append( " " ).append( XMLHandler.addTagValue( "compression", - ( fileCompression == null ) ? "None" : fileCompression ) ); + ( content.fileCompression == null ) ? "None" : content.fileCompression ) ); retval.append( " " ).append( Const.CR ); retval.append( " " ).append( Const.CR ); @@ -1245,8 +703,8 @@ public String getXML() { retval.append( " " ).append( Const.CR ); retval.append( " " ).append( Const.CR ); - for ( int i = 0; i < inputFields.length; i++ ) { - TextFileInputField field = inputFields[i]; + for ( int i = 0; i < inputFiles.inputFields.length; i++ ) { + TextFileInputField field = inputFiles.inputFields[i]; retval.append( " " ).append( Const.CR ); retval.append( " " ).append( XMLHandler.addTagValue( "name", field.getName() ) ); @@ -1265,47 +723,47 @@ public String getXML() { retval.append( " " ).append( Const.CR ); } retval.append( " " ).append( Const.CR ); - retval.append( " " ).append( XMLHandler.addTagValue( "limit", rowLimit ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "limit", content.rowLimit ) ); // ERROR HANDLING - retval.append( " " ).append( XMLHandler.addTagValue( "error_ignored", errorIgnored ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "skip_bad_files", skipBadFiles ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "file_error_field", fileErrorField ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "file_error_message_field", fileErrorMessageField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_ignored", errorHandling.errorIgnored ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "skip_bad_files", errorHandling.skipBadFiles ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_error_field", errorHandling.fileErrorField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "file_error_message_field", errorHandling.fileErrorMessageField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "error_line_skipped", errorLineSkipped ) ); retval.append( " " ).append( XMLHandler.addTagValue( "error_count_field", errorCountField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "error_fields_field", errorFieldsField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "error_text_field", errorTextField ) ); retval.append( " " ).append( - XMLHandler.addTagValue( "bad_line_files_destination_directory", warningFilesDestinationDirectory ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "bad_line_files_extension", warningFilesExtension ) ); + XMLHandler.addTagValue( "bad_line_files_destination_directory",errorHandling. warningFilesDestinationDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "bad_line_files_extension",errorHandling. warningFilesExtension ) ); retval.append( " " ).append( - XMLHandler.addTagValue( "error_line_files_destination_directory", errorFilesDestinationDirectory ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "error_line_files_extension", errorFilesExtension ) ); + XMLHandler.addTagValue( "error_line_files_destination_directory",errorHandling. errorFilesDestinationDirectory ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "error_line_files_extension", errorHandling.errorFilesExtension ) ); retval.append( " " ).append( - XMLHandler.addTagValue( "line_number_files_destination_directory", lineNumberFilesDestinationDirectory ) ); + XMLHandler.addTagValue( "line_number_files_destination_directory", errorHandling.lineNumberFilesDestinationDirectory ) ); retval.append( " " ).append( - XMLHandler.addTagValue( "line_number_files_extension", lineNumberFilesExtension ) ); + XMLHandler.addTagValue( "line_number_files_extension",errorHandling. lineNumberFilesExtension ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "date_format_lenient", dateFormatLenient ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "date_format_locale", dateFormatLocale.toString() ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "date_format_lenient", content.dateFormatLenient ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "date_format_locale", content.dateFormatLocale.toString() ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "shortFileFieldName", shortFileFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "pathFieldName", pathFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "hiddenFieldName", hiddenFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "shortFileFieldName", additionalOutputFields.shortFilenameField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "pathFieldName",additionalOutputFields.pathField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "hiddenFieldName", additionalOutputFields.hiddenField ) ); retval.append( " " ).append( - XMLHandler.addTagValue( "lastModificationTimeFieldName", lastModificationTimeFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "uriNameFieldName", uriNameFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "rootUriNameFieldName", rootUriNameFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "extensionFieldName", extensionFieldName ) ); - retval.append( " " ).append( XMLHandler.addTagValue( "sizeFieldName", sizeFieldName ) ); + XMLHandler.addTagValue( "lastModificationTimeFieldName",additionalOutputFields.lastModificationField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "uriNameFieldName",additionalOutputFields.uriField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "rootUriNameFieldName",additionalOutputFields.rootUriField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "extensionFieldName", additionalOutputFields.extensionField ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "sizeFieldName", additionalOutputFields.sizeField ) ); return retval.toString(); } public String getLookupStepname() { - if ( acceptingFilenames && acceptingStep != null && !Const.isEmpty( acceptingStep.getName() ) ) { + if ( inputFiles.acceptingFilenames && acceptingStep != null && !Const.isEmpty( acceptingStep.getName() ) ) { return acceptingStep.getName(); } return null; @@ -1316,11 +774,11 @@ public String getLookupStepname() { * optionally search the info step in a list of steps */ public void searchInfoAndTargetSteps( List steps ) { - acceptingStep = StepMeta.findStep( steps, acceptingStepName ); + acceptingStep = StepMeta.findStep( steps, inputFiles.acceptingStepName ); } public String[] getInfoSteps() { - if ( acceptingFilenames && acceptingStep != null ) { + if (inputFiles. acceptingFilenames && acceptingStep != null ) { return new String[] { acceptingStep.getName() }; } return null; @@ -1328,42 +786,42 @@ public String[] getInfoSteps() { public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List databases ) throws KettleException { try { - acceptingFilenames = rep.getStepAttributeBoolean( id_step, "accept_filenames" ); - passingThruFields = rep.getStepAttributeBoolean( id_step, "passing_through_fields" ); - acceptingField = rep.getStepAttributeString( id_step, "accept_field" ); - acceptingStepName = rep.getStepAttributeString( id_step, "accept_stepname" ); - - separator = rep.getStepAttributeString( id_step, "separator" ); - enclosure = rep.getStepAttributeString( id_step, "enclosure" ); - breakInEnclosureAllowed = rep.getStepAttributeBoolean( id_step, "enclosure_breaks" ); - escapeCharacter = rep.getStepAttributeString( id_step, "escapechar" ); - header = rep.getStepAttributeBoolean( id_step, "header" ); - nrHeaderLines = (int) rep.getStepAttributeInteger( id_step, "nr_headerlines" ); - footer = rep.getStepAttributeBoolean( id_step, "footer" ); - nrFooterLines = (int) rep.getStepAttributeInteger( id_step, "nr_footerlines" ); - lineWrapped = rep.getStepAttributeBoolean( id_step, "line_wrapped" ); - nrWraps = (int) rep.getStepAttributeInteger( id_step, "nr_wraps" ); - layoutPaged = rep.getStepAttributeBoolean( id_step, "layout_paged" ); - nrLinesPerPage = (int) rep.getStepAttributeInteger( id_step, "nr_lines_per_page" ); - nrLinesDocHeader = (int) rep.getStepAttributeInteger( id_step, "nr_lines_doc_header" ); - noEmptyLines = rep.getStepAttributeBoolean( id_step, "noempty" ); - - includeFilename = rep.getStepAttributeBoolean( id_step, "include" ); - filenameField = rep.getStepAttributeString( id_step, "include_field" ); - includeRowNumber = rep.getStepAttributeBoolean( id_step, "rownum" ); - rowNumberByFile = rep.getStepAttributeBoolean( id_step, "rownumByFile" ); - rowNumberField = rep.getStepAttributeString( id_step, "rownum_field" ); - - fileFormat = rep.getStepAttributeString( id_step, "format" ); - encoding = rep.getStepAttributeString( id_step, "encoding" ); + inputFiles. acceptingFilenames = rep.getStepAttributeBoolean( id_step, "accept_filenames" ); + inputFiles.passingThruFields = rep.getStepAttributeBoolean( id_step, "passing_through_fields" ); + inputFiles. acceptingField = rep.getStepAttributeString( id_step, "accept_field" ); + inputFiles. acceptingStepName = rep.getStepAttributeString( id_step, "accept_stepname" ); + + content.separator = rep.getStepAttributeString( id_step, "separator" ); + content.enclosure = rep.getStepAttributeString( id_step, "enclosure" ); + content.breakInEnclosureAllowed = rep.getStepAttributeBoolean( id_step, "enclosure_breaks" ); + content.escapeCharacter = rep.getStepAttributeString( id_step, "escapechar" ); + content.header = rep.getStepAttributeBoolean( id_step, "header" ); + content.nrHeaderLines = (int) rep.getStepAttributeInteger( id_step, "nr_headerlines" ); + content.footer = rep.getStepAttributeBoolean( id_step, "footer" ); + content.nrFooterLines = (int) rep.getStepAttributeInteger( id_step, "nr_footerlines" ); + content.lineWrapped = rep.getStepAttributeBoolean( id_step, "line_wrapped" ); + content.nrWraps = (int) rep.getStepAttributeInteger( id_step, "nr_wraps" ); + content.layoutPaged = rep.getStepAttributeBoolean( id_step, "layout_paged" ); + content.nrLinesPerPage = (int) rep.getStepAttributeInteger( id_step, "nr_lines_per_page" ); + content.nrLinesDocHeader = (int) rep.getStepAttributeInteger( id_step, "nr_lines_doc_header" ); + content.noEmptyLines = rep.getStepAttributeBoolean( id_step, "noempty" ); + + content.includeFilename = rep.getStepAttributeBoolean( id_step, "include" ); + content.filenameField = rep.getStepAttributeString( id_step, "include_field" ); + content.includeRowNumber = rep.getStepAttributeBoolean( id_step, "rownum" ); + content.rowNumberByFile = rep.getStepAttributeBoolean( id_step, "rownumByFile" ); + content.rowNumberField = rep.getStepAttributeString( id_step, "rownum_field" ); + + content.fileFormat = rep.getStepAttributeString( id_step, "format" ); + content. encoding = rep.getStepAttributeString( id_step, "encoding" ); String addToResult = rep.getStepAttributeString( id_step, "add_to_result_filenames" ); if ( Const.isEmpty( addToResult ) ) { - isaddresult = true; + inputFiles.isaddresult = true; } else { - isaddresult = rep.getStepAttributeBoolean( id_step, "add_to_result_filenames" ); + inputFiles. isaddresult = rep.getStepAttributeBoolean( id_step, "add_to_result_filenames" ); } - rowLimit = rep.getStepAttributeInteger( id_step, "limit" ); + content.rowLimit = rep.getStepAttributeInteger( id_step, "limit" ); int nrfiles = rep.countNrStepAttributes( id_step, "file_name" ); int nrfields = rep.countNrStepAttributes( id_step, "field_name" ); @@ -1372,24 +830,24 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis allocate( nrfiles, nrfields, nrfilters ); for ( int i = 0; i < nrfiles; i++ ) { - fileName[i] = loadSourceRep( rep, id_step, i ); - fileMask[i] = rep.getStepAttributeString( id_step, i, "file_mask" ); - excludeFileMask[i] = rep.getStepAttributeString( id_step, i, "exclude_file_mask" ); - fileRequired[i] = rep.getStepAttributeString( id_step, i, "file_required" ); - if ( !YES.equalsIgnoreCase( fileRequired[i] ) ) { - fileRequired[i] = NO; + inputFiles.fileName[i] = loadSourceRep( rep, id_step, i ); + inputFiles.fileMask[i] = rep.getStepAttributeString( id_step, i, "file_mask" ); + inputFiles.excludeFileMask[i] = rep.getStepAttributeString( id_step, i, "exclude_file_mask" ); + inputFiles.fileRequired[i] = rep.getStepAttributeString( id_step, i, "file_required" ); + if ( !YES.equalsIgnoreCase(inputFiles. fileRequired[i] ) ) { + inputFiles. fileRequired[i] = NO; } - includeSubFolders[i] = rep.getStepAttributeString( id_step, i, "include_subfolders" ); - if ( !YES.equalsIgnoreCase( includeSubFolders[i] ) ) { - includeSubFolders[i] = NO; + inputFiles. includeSubFolders[i] = rep.getStepAttributeString( id_step, i, "include_subfolders" ); + if ( !YES.equalsIgnoreCase( inputFiles.includeSubFolders[i] ) ) { + inputFiles.includeSubFolders[i] = NO; } } - fileType = rep.getStepAttributeString( id_step, "file_type" ); - fileCompression = rep.getStepAttributeString( id_step, "compression" ); - if ( fileCompression == null ) { - fileCompression = "None"; + content.fileType = rep.getStepAttributeString( id_step, "file_type" ); + content.fileCompression = rep.getStepAttributeString( id_step, "compression" ); + if ( content.fileCompression == null ) { + content.fileCompression = "None"; if ( rep.getStepAttributeBoolean( id_step, "file_zipped" ) ) { - fileCompression = "Zip"; + content.fileCompression = "Zip"; } } @@ -1419,42 +877,42 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis .getTrimTypeByCode( rep.getStepAttributeString( id_step, i, "field_trim_type" ) ) ); field.setRepeated( rep.getStepAttributeBoolean( id_step, i, "field_repeat" ) ); - inputFields[i] = field; + inputFiles. inputFields[i] = field; } - errorIgnored = rep.getStepAttributeBoolean( id_step, "error_ignored" ); - skipBadFiles = rep.getStepAttributeBoolean( id_step, "skip_bad_files" ); - fileErrorField = rep.getStepAttributeString( id_step, "file_error_field" ); - fileErrorMessageField = rep.getStepAttributeString( id_step, "file_error_message_field" ); + errorHandling.errorIgnored = rep.getStepAttributeBoolean( id_step, "error_ignored" ); + errorHandling.skipBadFiles = rep.getStepAttributeBoolean( id_step, "skip_bad_files" ); + errorHandling.fileErrorField = rep.getStepAttributeString( id_step, "file_error_field" ); + errorHandling.fileErrorMessageField = rep.getStepAttributeString( id_step, "file_error_message_field" ); errorLineSkipped = rep.getStepAttributeBoolean( id_step, "error_line_skipped" ); errorCountField = rep.getStepAttributeString( id_step, "error_count_field" ); errorFieldsField = rep.getStepAttributeString( id_step, "error_fields_field" ); errorTextField = rep.getStepAttributeString( id_step, "error_text_field" ); - warningFilesDestinationDirectory = rep.getStepAttributeString( id_step, "bad_line_files_dest_dir" ); - warningFilesExtension = rep.getStepAttributeString( id_step, "bad_line_files_ext" ); - errorFilesDestinationDirectory = rep.getStepAttributeString( id_step, "error_line_files_dest_dir" ); - errorFilesExtension = rep.getStepAttributeString( id_step, "error_line_files_ext" ); - lineNumberFilesDestinationDirectory = rep.getStepAttributeString( id_step, "line_number_files_dest_dir" ); - lineNumberFilesExtension = rep.getStepAttributeString( id_step, "line_number_files_ext" ); + errorHandling. warningFilesDestinationDirectory = rep.getStepAttributeString( id_step, "bad_line_files_dest_dir" ); + errorHandling. warningFilesExtension = rep.getStepAttributeString( id_step, "bad_line_files_ext" ); + errorHandling. errorFilesDestinationDirectory = rep.getStepAttributeString( id_step, "error_line_files_dest_dir" ); + errorHandling. errorFilesExtension = rep.getStepAttributeString( id_step, "error_line_files_ext" ); + errorHandling. lineNumberFilesDestinationDirectory = rep.getStepAttributeString( id_step, "line_number_files_dest_dir" ); + errorHandling. lineNumberFilesExtension = rep.getStepAttributeString( id_step, "line_number_files_ext" ); - dateFormatLenient = rep.getStepAttributeBoolean( id_step, 0, "date_format_lenient", true ); + content.dateFormatLenient = rep.getStepAttributeBoolean( id_step, 0, "date_format_lenient", true ); String dateLocale = rep.getStepAttributeString( id_step, 0, "date_format_locale" ); if ( dateLocale != null ) { - dateFormatLocale = EnvUtil.createLocale( dateLocale ); + content.dateFormatLocale = EnvUtil.createLocale( dateLocale ); } else { - dateFormatLocale = Locale.getDefault(); + content.dateFormatLocale = Locale.getDefault(); } - shortFileFieldName = rep.getStepAttributeString( id_step, "shortFileFieldName" ); - pathFieldName = rep.getStepAttributeString( id_step, "pathFieldName" ); - hiddenFieldName = rep.getStepAttributeString( id_step, "hiddenFieldName" ); - lastModificationTimeFieldName = rep.getStepAttributeString( id_step, "lastModificationTimeFieldName" ); - uriNameFieldName = rep.getStepAttributeString( id_step, "uriNameFieldName" ); - rootUriNameFieldName = rep.getStepAttributeString( id_step, "rootUriNameFieldName" ); - extensionFieldName = rep.getStepAttributeString( id_step, "extensionFieldName" ); - sizeFieldName = rep.getStepAttributeString( id_step, "sizeFieldName" ); + additionalOutputFields.shortFilenameField = rep.getStepAttributeString( id_step, "shortFileFieldName" ); + additionalOutputFields.pathField = rep.getStepAttributeString( id_step, "pathFieldName" ); + additionalOutputFields.hiddenField = rep.getStepAttributeString( id_step, "hiddenFieldName" ); + additionalOutputFields.lastModificationField = rep.getStepAttributeString( id_step, "lastModificationTimeFieldName" ); + additionalOutputFields.uriField = rep.getStepAttributeString( id_step, "uriNameFieldName" ); + additionalOutputFields.rootUriField = rep.getStepAttributeString( id_step, "rootUriNameFieldName" ); + additionalOutputFields.extensionField = rep.getStepAttributeString( id_step, "extensionFieldName" ); + additionalOutputFields.sizeField = rep.getStepAttributeString( id_step, "sizeFieldName" ); } catch ( Exception e ) { throw new KettleException( "Unexpected error reading step information from the repository", e ); } @@ -1462,50 +920,50 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step ) throws KettleException { try { - rep.saveStepAttribute( id_transformation, id_step, "accept_filenames", acceptingFilenames ); - rep.saveStepAttribute( id_transformation, id_step, "passing_through_fields", passingThruFields ); - rep.saveStepAttribute( id_transformation, id_step, "accept_field", acceptingField ); + rep.saveStepAttribute( id_transformation, id_step, "accept_filenames", inputFiles.acceptingFilenames ); + rep.saveStepAttribute( id_transformation, id_step, "passing_through_fields", inputFiles.passingThruFields ); + rep.saveStepAttribute( id_transformation, id_step, "accept_field",inputFiles. acceptingField ); rep.saveStepAttribute( id_transformation, id_step, "accept_stepname", ( acceptingStep != null ? acceptingStep.getName() : "" ) ); - rep.saveStepAttribute( id_transformation, id_step, "separator", separator ); - rep.saveStepAttribute( id_transformation, id_step, "enclosure", enclosure ); - rep.saveStepAttribute( id_transformation, id_step, "enclosure_breaks", breakInEnclosureAllowed ); - rep.saveStepAttribute( id_transformation, id_step, "escapechar", escapeCharacter ); - rep.saveStepAttribute( id_transformation, id_step, "header", header ); - rep.saveStepAttribute( id_transformation, id_step, "nr_headerlines", nrHeaderLines ); - rep.saveStepAttribute( id_transformation, id_step, "footer", footer ); - rep.saveStepAttribute( id_transformation, id_step, "nr_footerlines", nrFooterLines ); - rep.saveStepAttribute( id_transformation, id_step, "line_wrapped", lineWrapped ); - rep.saveStepAttribute( id_transformation, id_step, "nr_wraps", nrWraps ); - rep.saveStepAttribute( id_transformation, id_step, "layout_paged", layoutPaged ); - rep.saveStepAttribute( id_transformation, id_step, "nr_lines_per_page", nrLinesPerPage ); - rep.saveStepAttribute( id_transformation, id_step, "nr_lines_doc_header", nrLinesDocHeader ); - - rep.saveStepAttribute( id_transformation, id_step, "noempty", noEmptyLines ); - - rep.saveStepAttribute( id_transformation, id_step, "include", includeFilename ); - rep.saveStepAttribute( id_transformation, id_step, "include_field", filenameField ); - rep.saveStepAttribute( id_transformation, id_step, "rownum", includeRowNumber ); - rep.saveStepAttribute( id_transformation, id_step, "rownumByFile", rowNumberByFile ); - rep.saveStepAttribute( id_transformation, id_step, "rownum_field", rowNumberField ); - - rep.saveStepAttribute( id_transformation, id_step, "format", fileFormat ); - rep.saveStepAttribute( id_transformation, id_step, "encoding", encoding ); - rep.saveStepAttribute( id_transformation, id_step, "add_to_result_filenames", isaddresult ); - - rep.saveStepAttribute( id_transformation, id_step, "limit", rowLimit ); - - for ( int i = 0; i < fileName.length; i++ ) { - saveSourceRep( rep, id_transformation, id_step, i, fileName[i] ); - rep.saveStepAttribute( id_transformation, id_step, i, "file_mask", fileMask[i] ); - rep.saveStepAttribute( id_transformation, id_step, i, "exclude_file_mask", excludeFileMask[i] ); - rep.saveStepAttribute( id_transformation, id_step, i, "file_required", fileRequired[i] ); - rep.saveStepAttribute( id_transformation, id_step, i, "include_subfolders", includeSubFolders[i] ); + rep.saveStepAttribute( id_transformation, id_step, "separator", content.separator ); + rep.saveStepAttribute( id_transformation, id_step, "enclosure", content.enclosure ); + rep.saveStepAttribute( id_transformation, id_step, "enclosure_breaks", content.breakInEnclosureAllowed ); + rep.saveStepAttribute( id_transformation, id_step, "escapechar", content.escapeCharacter ); + rep.saveStepAttribute( id_transformation, id_step, "header", content.header ); + rep.saveStepAttribute( id_transformation, id_step, "nr_headerlines",content. nrHeaderLines ); + rep.saveStepAttribute( id_transformation, id_step, "footer",content. footer ); + rep.saveStepAttribute( id_transformation, id_step, "nr_footerlines", content.nrFooterLines ); + rep.saveStepAttribute( id_transformation, id_step, "line_wrapped", content.lineWrapped ); + rep.saveStepAttribute( id_transformation, id_step, "nr_wraps", content.nrWraps ); + rep.saveStepAttribute( id_transformation, id_step, "layout_paged", content.layoutPaged ); + rep.saveStepAttribute( id_transformation, id_step, "nr_lines_per_page", content.nrLinesPerPage ); + rep.saveStepAttribute( id_transformation, id_step, "nr_lines_doc_header", content.nrLinesDocHeader ); + + rep.saveStepAttribute( id_transformation, id_step, "noempty",content. noEmptyLines ); + + rep.saveStepAttribute( id_transformation, id_step, "include", content.includeFilename ); + rep.saveStepAttribute( id_transformation, id_step, "include_field", content.filenameField ); + rep.saveStepAttribute( id_transformation, id_step, "rownum", content.includeRowNumber ); + rep.saveStepAttribute( id_transformation, id_step, "rownumByFile", content.rowNumberByFile ); + rep.saveStepAttribute( id_transformation, id_step, "rownum_field", content.rowNumberField ); + + rep.saveStepAttribute( id_transformation, id_step, "format", content.fileFormat ); + rep.saveStepAttribute( id_transformation, id_step, "encoding", content.encoding ); + rep.saveStepAttribute( id_transformation, id_step, "add_to_result_filenames",inputFiles. isaddresult ); + + rep.saveStepAttribute( id_transformation, id_step, "limit", content.rowLimit ); + + for ( int i = 0; i < inputFiles.fileName.length; i++ ) { + saveSourceRep( rep, id_transformation, id_step, i, inputFiles.fileName[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "file_mask", inputFiles.fileMask[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "exclude_file_mask", inputFiles.excludeFileMask[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "file_required", inputFiles.fileRequired[i] ); + rep.saveStepAttribute( id_transformation, id_step, i, "include_subfolders",inputFiles. includeSubFolders[i] ); } - rep.saveStepAttribute( id_transformation, id_step, "file_type", fileType ); + rep.saveStepAttribute( id_transformation, id_step, "file_type", content.fileType ); rep.saveStepAttribute( id_transformation, id_step, "compression", - ( fileCompression == null ) ? "None" : fileCompression ); + ( content.fileCompression == null ) ? "None" : content.fileCompression ); for ( int i = 0; i < filter.length; i++ ) { rep.saveStepAttribute( id_transformation, id_step, i, "filter_position", filter[i].getFilterPosition() ); @@ -1514,8 +972,8 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform rep.saveStepAttribute( id_transformation, id_step, i, "filter_is_positive", filter[i].isFilterPositive() ); } - for ( int i = 0; i < inputFields.length; i++ ) { - TextFileInputField field = inputFields[i]; + for ( int i = 0; i < inputFiles.inputFields.length; i++ ) { + TextFileInputField field = inputFiles.inputFields[i]; rep.saveStepAttribute( id_transformation, id_step, i, "field_name", field.getName() ); rep.saveStepAttribute( id_transformation, id_step, i, "field_type", field.getTypeDesc() ); @@ -1532,61 +990,42 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform rep.saveStepAttribute( id_transformation, id_step, i, "field_repeat", field.isRepeated() ); } - rep.saveStepAttribute( id_transformation, id_step, "error_ignored", errorIgnored ); - rep.saveStepAttribute( id_transformation, id_step, "skip_bad_files", skipBadFiles ); - rep.saveStepAttribute( id_transformation, id_step, "file_error_field", fileErrorField ); - rep.saveStepAttribute( id_transformation, id_step, "file_error_message_field", fileErrorMessageField ); + rep.saveStepAttribute( id_transformation, id_step, "error_ignored", errorHandling.errorIgnored ); + rep.saveStepAttribute( id_transformation, id_step, "skip_bad_files", errorHandling.skipBadFiles ); + rep.saveStepAttribute( id_transformation, id_step, "file_error_field", errorHandling.fileErrorField ); + rep.saveStepAttribute( id_transformation, id_step, "file_error_message_field",errorHandling. fileErrorMessageField ); rep.saveStepAttribute( id_transformation, id_step, "error_line_skipped", errorLineSkipped ); rep.saveStepAttribute( id_transformation, id_step, "error_count_field", errorCountField ); rep.saveStepAttribute( id_transformation, id_step, "error_fields_field", errorFieldsField ); rep.saveStepAttribute( id_transformation, id_step, "error_text_field", errorTextField ); rep.saveStepAttribute( - id_transformation, id_step, "bad_line_files_dest_dir", warningFilesDestinationDirectory ); - rep.saveStepAttribute( id_transformation, id_step, "bad_line_files_ext", warningFilesExtension ); + id_transformation, id_step, "bad_line_files_dest_dir",errorHandling. warningFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "bad_line_files_ext", errorHandling.warningFilesExtension ); rep.saveStepAttribute( - id_transformation, id_step, "error_line_files_dest_dir", errorFilesDestinationDirectory ); - rep.saveStepAttribute( id_transformation, id_step, "error_line_files_ext", errorFilesExtension ); + id_transformation, id_step, "error_line_files_dest_dir", errorHandling.errorFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "error_line_files_ext",errorHandling. errorFilesExtension ); rep.saveStepAttribute( - id_transformation, id_step, "line_number_files_dest_dir", lineNumberFilesDestinationDirectory ); - rep.saveStepAttribute( id_transformation, id_step, "line_number_files_ext", lineNumberFilesExtension ); + id_transformation, id_step, "line_number_files_dest_dir", errorHandling.lineNumberFilesDestinationDirectory ); + rep.saveStepAttribute( id_transformation, id_step, "line_number_files_ext", errorHandling.lineNumberFilesExtension ); - rep.saveStepAttribute( id_transformation, id_step, "date_format_lenient", dateFormatLenient ); - rep.saveStepAttribute( id_transformation, id_step, "date_format_locale", dateFormatLocale.toString() ); + rep.saveStepAttribute( id_transformation, id_step, "date_format_lenient", content.dateFormatLenient ); + rep.saveStepAttribute( id_transformation, id_step, "date_format_locale", content.dateFormatLocale.toString() ); - rep.saveStepAttribute( id_transformation, id_step, "shortFileFieldName", shortFileFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "pathFieldName", pathFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "hiddenFieldName", hiddenFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "shortFileFieldName",additionalOutputFields.shortFilenameField ); + rep.saveStepAttribute( id_transformation, id_step, "pathFieldName", additionalOutputFields.pathField ); + rep.saveStepAttribute( id_transformation, id_step, "hiddenFieldName",additionalOutputFields.hiddenField ); rep.saveStepAttribute( - id_transformation, id_step, "lastModificationTimeFieldName", lastModificationTimeFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "uriNameFieldName", uriNameFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "rootUriNameFieldName", rootUriNameFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "extensionFieldName", extensionFieldName ); - rep.saveStepAttribute( id_transformation, id_step, "sizeFieldName", sizeFieldName ); + id_transformation, id_step, "lastModificationTimeFieldName",additionalOutputFields.lastModificationField ); + rep.saveStepAttribute( id_transformation, id_step, "uriNameFieldName", additionalOutputFields.uriField ); + rep.saveStepAttribute( id_transformation, id_step, "rootUriNameFieldName", additionalOutputFields.rootUriField ); + rep.saveStepAttribute( id_transformation, id_step, "extensionFieldName",additionalOutputFields.extensionField ); + rep.saveStepAttribute( id_transformation, id_step, "sizeFieldName", additionalOutputFields.sizeField ); } catch ( Exception e ) { throw new KettleException( "Unable to save step information to the repository for id_step=" + id_step, e ); } } - public String[] getFilePaths( VariableSpace space ) { - return FileInputList.createFilePathList( - space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); - } - - public FileInputList getTextFileList( VariableSpace space ) { - return FileInputList.createFileList( - space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); - } - - private boolean[] includeSubFolderBoolean() { - int len = fileName.length; - boolean[] includeSubFolderBoolean = new boolean[len]; - for ( int i = 0; i < len; i++ ) { - includeSubFolderBoolean[i] = YES.equalsIgnoreCase( includeSubFolders[i] ); - } - return includeSubFolderBoolean; - } - public void check( List remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore ) { @@ -1594,7 +1033,7 @@ public void check( List remarks, TransMeta transMeta, Step // See if we get input... if ( input.length > 0 ) { - if ( !isAcceptingFilenames() ) { + if ( !inputFiles.acceptingFilenames ) { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString( PKG, "TextFileInputMeta.CheckResult.NoInputError" ), stepMeta ); @@ -1614,7 +1053,7 @@ public void check( List remarks, TransMeta transMeta, Step FileInputList textFileList = getTextFileList( transMeta ); if ( textFileList.nrOfFiles() == 0 ) { - if ( !isAcceptingFilenames() ) { + if ( !inputFiles.acceptingFilenames ) { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString( PKG, "TextFileInputMeta.CheckResult.ExpectedFilesError" ), stepMeta ); @@ -1637,21 +1076,6 @@ public StepDataInterface getStepData() { return new TextFileInputData(); } - /** - * @return Returns the escapeCharacter. - */ - public String getEscapeCharacter() { - return escapeCharacter; - } - - /** - * @param escapeCharacter - * The escapeCharacter to set. - */ - public void setEscapeCharacter( String escapeCharacter ) { - this.escapeCharacter = escapeCharacter; - } - public String getErrorCountField() { return errorCountField; } @@ -1668,14 +1092,6 @@ public void setErrorFieldsField( String errorFieldsField ) { this.errorFieldsField = errorFieldsField; } - public boolean isErrorIgnored() { - return errorIgnored; - } - - public void setErrorIgnored( boolean errorIgnored ) { - this.errorIgnored = errorIgnored; - } - public String getErrorTextField() { return errorTextField; } @@ -1684,35 +1100,6 @@ public void setErrorTextField( String errorTextField ) { this.errorTextField = errorTextField; } - /** - * @return Returns the lineWrapped. - */ - public boolean isLineWrapped() { - return lineWrapped; - } - - /** - * @param lineWrapped - * The lineWrapped to set. - */ - public void setLineWrapped( boolean lineWrapped ) { - this.lineWrapped = lineWrapped; - } - - /** - * @return Returns the nrFooterLines. - */ - public int getNrFooterLines() { - return nrFooterLines; - } - - /** - * @param nrFooterLines - * The nrFooterLines to set. - */ - public void setNrFooterLines( int nrFooterLines ) { - this.nrFooterLines = nrFooterLines; - } public String getRequiredFilesDesc( String tt ) { if ( tt == null ) { @@ -1725,151 +1112,6 @@ public String getRequiredFilesDesc( String tt ) { } } - /** - * @return Returns the nrHeaderLines. - */ - public int getNrHeaderLines() { - return nrHeaderLines; - } - - /** - * @param nrHeaderLines - * The nrHeaderLines to set. - */ - public void setNrHeaderLines( int nrHeaderLines ) { - this.nrHeaderLines = nrHeaderLines; - } - - /** - * @return Returns the nrWraps. - */ - public int getNrWraps() { - return nrWraps; - } - - /** - * @param nrWraps - * The nrWraps to set. - */ - public void setNrWraps( int nrWraps ) { - this.nrWraps = nrWraps; - } - - /** - * @return Returns the layoutPaged. - */ - public boolean isLayoutPaged() { - return layoutPaged; - } - - /** - * @param layoutPaged - * The layoutPaged to set. - */ - public void setLayoutPaged( boolean layoutPaged ) { - this.layoutPaged = layoutPaged; - } - - /** - * @return Returns the nrLinesPerPage. - */ - public int getNrLinesPerPage() { - return nrLinesPerPage; - } - - /** - * @param nrLinesPerPage - * The nrLinesPerPage to set. - */ - public void setNrLinesPerPage( int nrLinesPerPage ) { - this.nrLinesPerPage = nrLinesPerPage; - } - - /** - * @return Returns the nrLinesDocHeader. - */ - public int getNrLinesDocHeader() { - return nrLinesDocHeader; - } - - /** - * @param nrLinesDocHeader - * The nrLinesDocHeader to set. - */ - public void setNrLinesDocHeader( int nrLinesDocHeader ) { - this.nrLinesDocHeader = nrLinesDocHeader; - } - - public String getWarningFilesDestinationDirectory() { - return warningFilesDestinationDirectory; - } - - public void setWarningFilesDestinationDirectory( String warningFilesDestinationDirectory ) { - this.warningFilesDestinationDirectory = warningFilesDestinationDirectory; - } - - public String getWarningFilesExtension() { - return warningFilesExtension; - } - - public void setWarningFilesExtension( String warningFilesExtension ) { - this.warningFilesExtension = warningFilesExtension; - } - - public String getLineNumberFilesDestinationDirectory() { - return lineNumberFilesDestinationDirectory; - } - - public void setLineNumberFilesDestinationDirectory( String lineNumberFilesDestinationDirectory ) { - this.lineNumberFilesDestinationDirectory = lineNumberFilesDestinationDirectory; - } - - public String getLineNumberFilesExtension() { - return lineNumberFilesExtension; - } - - public void setLineNumberFilesExtension( String lineNumberFilesExtension ) { - this.lineNumberFilesExtension = lineNumberFilesExtension; - } - - public String getErrorFilesDestinationDirectory() { - return errorFilesDestinationDirectory; - } - - public void setErrorFilesDestinationDirectory( String errorFilesDestinationDirectory ) { - this.errorFilesDestinationDirectory = errorFilesDestinationDirectory; - } - - public String getErrorLineFilesExtension() { - return errorFilesExtension; - } - - public void setErrorLineFilesExtension( String errorLineFilesExtension ) { - this.errorFilesExtension = errorLineFilesExtension; - } - - public boolean isDateFormatLenient() { - return dateFormatLenient; - } - - public void setDateFormatLenient( boolean dateFormatLenient ) { - this.dateFormatLenient = dateFormatLenient; - } - - /** - * @param isaddresult - * The isaddresult to set. - */ - public void setAddResultFile( boolean isaddresult ) { - this.isaddresult = isaddresult; - } - - /** - * @return Returns isaddresult. - */ - public boolean isAddResultFile() { - return isaddresult; - } public boolean isErrorLineSkipped() { return errorLineSkipped; @@ -1879,67 +1121,6 @@ public void setErrorLineSkipped( boolean errorLineSkipped ) { this.errorLineSkipped = errorLineSkipped; } - /** - * @return Returns the dateFormatLocale. - */ - public Locale getDateFormatLocale() { - return dateFormatLocale; - } - - /** - * @param dateFormatLocale - * The dateFormatLocale to set. - */ - public void setDateFormatLocale( Locale dateFormatLocale ) { - this.dateFormatLocale = dateFormatLocale; - } - - public boolean isAcceptingFilenames() { - return acceptingFilenames; - } - - public void setAcceptingFilenames( boolean getFileFromJob ) { - this.acceptingFilenames = getFileFromJob; - } - - public boolean isPassingThruFields() { - return passingThruFields; - } - - public void setPassingThruFields( boolean passingThruFields ) { - this.passingThruFields = passingThruFields; - } - - /** - * @return Returns the fileNameField. - */ - public String getAcceptingField() { - return acceptingField; - } - - /** - * @param fileNameField - * The fileNameField to set. - */ - public void setAcceptingField( String fileNameField ) { - this.acceptingField = fileNameField; - } - - /** - * @return Returns the acceptingStep. - */ - public String getAcceptingStepName() { - return acceptingStepName; - } - - /** - * @param acceptingStep - * The acceptingStep to set. - */ - public void setAcceptingStepName( String acceptingStep ) { - this.acceptingStepName = acceptingStep; - } - /** * @return Returns the acceptingStep. */ @@ -1957,9 +1138,9 @@ public void setAcceptingStep( StepMeta acceptingStep ) { public int getFileFormatTypeNr() { // calculate the file format type in advance so we can use a switch - if ( getFileFormat().equalsIgnoreCase( "DOS" ) ) { + if ( content.fileFormat.equalsIgnoreCase( "DOS" ) ) { return FILE_FORMAT_DOS; - } else if ( getFileFormat().equalsIgnoreCase( "unix" ) ) { + } else if ( content.fileFormat.equalsIgnoreCase( "unix" ) ) { return TextFileInputMeta.FILE_FORMAT_UNIX; } else { return TextFileInputMeta.FILE_FORMAT_MIXED; @@ -1968,28 +1149,13 @@ public int getFileFormatTypeNr() { public int getFileTypeNr() { // calculate the file type in advance CSV or Fixed? - if ( getFileType().equalsIgnoreCase( "CSV" ) ) { + if ( content.fileType.equalsIgnoreCase( "CSV" ) ) { return TextFileInputMeta.FILE_TYPE_CSV; } else { return TextFileInputMeta.FILE_TYPE_FIXED; } } - @Override - public List getResourceDependencies( TransMeta transMeta, StepMeta stepInfo ) { - List references = new ArrayList( 5 ); - ResourceReference reference = new ResourceReference( stepInfo ); - references.add( reference ); - - String[] textFiles = getFilePaths( transMeta ); - if ( textFiles != null ) { - for ( int i = 0; i < textFiles.length; i++ ) { - reference.getEntries().add( new ResourceEntry( textFiles[i], ResourceType.FILE ) ); - } - } - return references; - } - /** * Since the exported transformation that runs this will reside in a ZIP file, we can't reference files relatively. So * what this does is turn the name of files into absolute paths OR it simply includes the resource in the ZIP file. @@ -2014,13 +1180,13 @@ public String exportResources( VariableSpace space, Map all ) throws break; case FILE_TYPE: - meta.setFileType( lookValue ); + meta.content.fileType = lookValue ; break; case SEPARATOR: - meta.setSeparator( lookValue ); + meta.content.separator= lookValue ; break; case ENCLOSURE: - meta.setEnclosure( lookValue ); + meta.content.enclosure= lookValue ; break; case ESCAPE_CHAR: - meta.setEscapeCharacter( lookValue ); + meta.content.escapeCharacter =lookValue ; break; case BREAK_IN_ENCLOSURE: - meta.setBreakInEnclosureAllowed( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.breakInEnclosureAllowed= "Y".equalsIgnoreCase( lookValue ) ; break; case HEADER_PRESENT: - meta.setHeader( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.header="Y".equalsIgnoreCase( lookValue ) ; break; case NR_HEADER_LINES: - meta.setNrHeaderLines( Const.toInt( lookValue, -1 ) ); + meta.content.nrHeaderLines = Const.toInt( lookValue, -1 ) ; break; case HAS_FOOTER: - meta.setFooter( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.footer = "Y".equalsIgnoreCase( lookValue ) ; break; case NR_FOOTER_LINES: - meta.setNrFooterLines( Const.toInt( lookValue, -1 ) ); + meta.content.nrFooterLines = Const.toInt( lookValue, -1 ) ; break; case HAS_WRAPPED_LINES: - meta.setLineWrapped( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.lineWrapped = "Y".equalsIgnoreCase( lookValue ) ; break; case NR_WRAPS: - meta.setNrWraps( Const.toInt( lookValue, -1 ) ); + meta.content.nrWraps = Const.toInt( lookValue, -1 ) ; break; case HAS_PAGED_LAYOUT: - meta.setLayoutPaged( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.layoutPaged = "Y".equalsIgnoreCase( lookValue ) ; break; case NR_DOC_HEADER_LINES: - meta.setNrLinesDocHeader( Const.toInt( lookValue, -1 ) ); + meta.content.nrLinesDocHeader = Const.toInt( lookValue, -1 ) ; break; case NR_LINES_PER_PAGE: - meta.setNrLinesPerPage( Const.toInt( lookValue, -1 ) ); + meta.content.nrLinesPerPage = Const.toInt( lookValue, -1 ) ; break; case COMPRESSION_TYPE: - meta.setFileCompression( lookValue ); + meta.content.fileCompression = lookValue ; break; case NO_EMPTY_LINES: - meta.setNoEmptyLines( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.noEmptyLines = "Y".equalsIgnoreCase( lookValue ) ; break; case INCLUDE_FILENAME: - meta.setIncludeFilename( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.includeFilename = "Y".equalsIgnoreCase( lookValue ) ; break; case FILENAME_FIELD: - meta.setFilenameField( lookValue ); + meta.content.filenameField = lookValue ; break; case INCLUDE_ROW_NUMBER: - meta.setIncludeRowNumber( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.includeRowNumber = "Y".equalsIgnoreCase( lookValue ) ; break; case ROW_NUMBER_BY_FILE: - meta.setRowNumberByFile( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.rowNumberByFile = "Y".equalsIgnoreCase( lookValue ) ; break; case ROW_NUMBER_FIELD: - meta.setRowNumberField( lookValue ); + meta.content.rowNumberField = lookValue ; break; case FILE_FORMAT: - meta.setFileFormat( lookValue ); + meta.content.fileFormat = lookValue ; break; case ENCODING: - meta.setEncoding( lookValue ); + meta.content.encoding = lookValue ; break; case ROW_LIMIT: - meta.setRowLimit( Const.toInt( lookValue, -1 ) ); + meta.content.rowLimit = Const.toInt( lookValue, -1 ) ; break; case DATE_FORMAT_LENIENT: - meta.setDateFormatLenient( "Y".equalsIgnoreCase( lookValue ) ); + meta.content.dateFormatLenient = "Y".equalsIgnoreCase( lookValue ) ; break; case DATE_FORMAT_LOCALE: - meta.setDateFormatLocale( new Locale( lookValue ) ); + meta.content.dateFormatLocale = new Locale( lookValue ) ; break; case ACCEPT_FILE_NAMES: - meta.setAcceptingFilenames( "Y".equalsIgnoreCase( lookValue ) ); + meta.inputFiles.acceptingFilenames= "Y".equalsIgnoreCase( lookValue ) ; break; case ACCEPT_FILE_STEP: - meta.setAcceptingStepName( lookValue ); + meta.inputFiles.acceptingStepName=lookValue ; break; case ACCEPT_FILE_FIELD: - meta.setAcceptingField( lookValue ); + meta.inputFiles.acceptingField= lookValue ; break; case PASS_THROUGH_FIELDS: - meta.setPassingThruFields( "Y".equalsIgnoreCase( lookValue ) ); + meta.inputFiles.passingThruFields= "Y".equalsIgnoreCase( lookValue ) ; break; case ADD_FILES_TO_RESULT: - meta.setAddResultFile( "Y".equalsIgnoreCase( lookValue ) ); + meta.inputFiles.isaddresult= "Y".equalsIgnoreCase( lookValue ) ; break; case FILE_SHORT_FILE_FIELDNAME: - meta.setShortFileNameField( lookValue ); + meta.additionalOutputFields.shortFilenameField= lookValue ; break; case FILE_PATH_FIELDNAME: - meta.setPathField( lookValue ); + meta.additionalOutputFields.pathField= lookValue ; break; case FILE_HIDDEN_FIELDNAME: - meta.setIsHiddenField( lookValue ); + meta.additionalOutputFields.hiddenField= lookValue ; break; case FILE_LAST_MODIFICATION_FIELDNAME: - meta.setLastModificationDateField( lookValue ); + meta.additionalOutputFields.lastModificationField= lookValue ; break; case FILE_URI_FIELDNAME: - meta.setUriField( lookValue ); + meta.additionalOutputFields.uriField= lookValue ; break; case FILE_EXTENSION_FIELDNAME: - meta.setExtensionField( lookValue ); + meta.additionalOutputFields.extensionField= lookValue ; break; case FILE_SIZE_FIELDNAME: - meta.setSizeField( lookValue ); + meta.additionalOutputFields.sizeField= lookValue ; break; case SKIP_BAD_FILES: - meta.setSkipBadFiles( "Y".equalsIgnoreCase( lookValue ) ); + meta.errorHandling.skipBadFiles= "Y".equalsIgnoreCase( lookValue ) ; break; case FILE_ERROR_FIELD: - meta.setFileErrorField( lookValue ); + meta.errorHandling.fileErrorField= lookValue ; break; case FILE_ERROR_MESSAGE_FIELD: - meta.setFileErrorMessageField( lookValue ); + meta.errorHandling.fileErrorMessageField= lookValue ; break; case IGNORE_ERRORS: - meta.setErrorIgnored( "Y".equalsIgnoreCase( lookValue ) ); + meta.errorHandling.errorIgnored= "Y".equalsIgnoreCase( lookValue ) ; break; case ERROR_COUNT_FIELD: meta.setErrorCountField( lookValue ); @@ -532,22 +532,22 @@ public void injectStepMetadataEntries( List all ) throws meta.setErrorTextField( lookValue ); break; case WARNING_FILES_TARGET_DIR: - meta.setWarningFilesDestinationDirectory( lookValue ); + meta.errorHandling.warningFilesDestinationDirectory= lookValue ; break; case WARNING_FILES_EXTENTION: - meta.setWarningFilesExtension( lookValue ); + meta.errorHandling.warningFilesExtension= lookValue ; break; case ERROR_FILES_TARGET_DIR: - meta.setErrorFilesDestinationDirectory( lookValue ); + meta.errorHandling.errorFilesDestinationDirectory=lookValue ; break; case ERROR_FILES_EXTENTION: - meta.setErrorLineFilesExtension( lookValue ); + meta.errorHandling.errorFilesExtension= lookValue ; break; case LINE_NR_FILES_TARGET_DIR: - meta.setLineNumberFilesDestinationDirectory( lookValue ); + meta.errorHandling.lineNumberFilesDestinationDirectory= lookValue ; break; case LINE_NR_FILES_EXTENTION: - meta.setLineNumberFilesExtension( lookValue ); + meta.errorHandling.lineNumberFilesExtension= lookValue ; break; case ERROR_LINES_SKIPPED: meta.setErrorLineSkipped( "Y".equalsIgnoreCase( lookValue ) ); @@ -561,7 +561,7 @@ public void injectStepMetadataEntries( List all ) throws // Only change a list when you need to, don't clear/reset existing content if you don't send new content. // if ( fields.size() > 0 ) { - meta.setInputFields( fields.toArray( new TextFileInputField[fields.size()] ) ); + meta.inputFiles.inputFields= fields.toArray( new TextFileInputField[fields.size()] ) ; } if ( fileLines.size() > 0 ) { meta.allocateFiles( fileLines.size() ); @@ -569,11 +569,10 @@ public void injectStepMetadataEntries( List all ) throws for ( int i = 0; i < fileLines.size(); i++ ) { FileLine fileLine = fileLines.get( i ); meta.getFileName()[i] = fileLine.filename; - meta.getFileMask()[i] = fileLine.includeMask; - meta.getExludeFileMask()[i] = fileLine.excludeMask; - meta.getExludeFileMask()[i] = fileLine.excludeMask; - meta.getFileRequired()[i] = fileLine.required; - meta.getIncludeSubFolders()[i] = fileLine.includeSubfolders; + meta.inputFiles.fileMask[i] = fileLine.includeMask; + meta.inputFiles.excludeFileMask[i] = fileLine.excludeMask; + meta.inputFiles.fileRequired[i] = fileLine.required; + meta.inputFiles.includeSubFolders[i] = fileLine.includeSubfolders; } } if ( filters.size() > 0 ) { @@ -584,57 +583,57 @@ public void injectStepMetadataEntries( List all ) throws public List extractStepMetadataEntries() throws KettleException { List result = new ArrayList(); - result.add( getEntry( Entry.FILE_TYPE, meta.getFileType() ) ); - result.add( getEntry( Entry.SEPARATOR, meta.getSeparator() ) ); - result.add( getEntry( Entry.ENCLOSURE, meta.getEnclosure() ) ); - result.add( getEntry( Entry.ESCAPE_CHAR, meta.getEscapeCharacter() ) ); - result.add( getEntry( Entry.BREAK_IN_ENCLOSURE, meta.isBreakInEnclosureAllowed() ) ); - result.add( getEntry( Entry.HEADER_PRESENT, meta.hasHeader() ) ); - result.add( getEntry( Entry.NR_HEADER_LINES, meta.getNrHeaderLines() ) ); - result.add( getEntry( Entry.HAS_FOOTER, meta.hasFooter() ) ); - result.add( getEntry( Entry.NR_FOOTER_LINES, meta.getNrFooterLines() ) ); - result.add( getEntry( Entry.HAS_WRAPPED_LINES, meta.isLineWrapped() ) ); - result.add( getEntry( Entry.NR_WRAPS, meta.getNrWraps() ) ); - result.add( getEntry( Entry.HAS_PAGED_LAYOUT, meta.isLayoutPaged() ) ); - result.add( getEntry( Entry.NR_DOC_HEADER_LINES, meta.getNrLinesDocHeader() ) ); - result.add( getEntry( Entry.NR_LINES_PER_PAGE, meta.getNrLinesPerPage() ) ); - result.add( getEntry( Entry.COMPRESSION_TYPE, meta.getFileCompression() ) ); - result.add( getEntry( Entry.NO_EMPTY_LINES, meta.noEmptyLines() ) ); - result.add( getEntry( Entry.INCLUDE_FILENAME, meta.includeFilename() ) ); - result.add( getEntry( Entry.FILENAME_FIELD, meta.getFilenameField() ) ); - result.add( getEntry( Entry.INCLUDE_ROW_NUMBER, meta.includeRowNumber() ) ); - result.add( getEntry( Entry.ROW_NUMBER_BY_FILE, meta.isRowNumberByFile() ) ); - result.add( getEntry( Entry.ROW_NUMBER_FIELD, meta.getRowNumberField() ) ); - result.add( getEntry( Entry.FILE_FORMAT, meta.getFileFormat() ) ); - result.add( getEntry( Entry.ENCODING, meta.getEncoding() ) ); - result.add( getEntry( Entry.ROW_LIMIT, meta.getRowLimit() ) ); - result.add( getEntry( Entry.DATE_FORMAT_LENIENT, meta.isDateFormatLenient() ) ); - result.add( getEntry( Entry.DATE_FORMAT_LOCALE, meta.getDateFormatLocale() ) ); - result.add( getEntry( Entry.ACCEPT_FILE_NAMES, meta.isAcceptingFilenames() ) ); - result.add( getEntry( Entry.ACCEPT_FILE_STEP, meta.getAcceptingStepName() ) ); - result.add( getEntry( Entry.ACCEPT_FILE_FIELD, meta.getAcceptingField() ) ); - result.add( getEntry( Entry.PASS_THROUGH_FIELDS, meta.isPassingThruFields() ) ); - result.add( getEntry( Entry.ADD_FILES_TO_RESULT, meta.isAddResultFile() ) ); - result.add( getEntry( Entry.FILE_SHORT_FILE_FIELDNAME, meta.getShortFileNameField() ) ); - result.add( getEntry( Entry.FILE_PATH_FIELDNAME, meta.getPathField() ) ); - result.add( getEntry( Entry.FILE_HIDDEN_FIELDNAME, meta.isHiddenField() ) ); - result.add( getEntry( Entry.FILE_LAST_MODIFICATION_FIELDNAME, meta.getLastModificationDateField() ) ); - result.add( getEntry( Entry.FILE_URI_FIELDNAME, meta.getUriField() ) ); - result.add( getEntry( Entry.FILE_EXTENSION_FIELDNAME, meta.getExtensionField() ) ); - result.add( getEntry( Entry.FILE_SIZE_FIELDNAME, meta.getSizeField() ) ); - result.add( getEntry( Entry.SKIP_BAD_FILES, meta.isSkipBadFiles() ) ); - result.add( getEntry( Entry.FILE_ERROR_FIELD, meta.getFileErrorField() ) ); - result.add( getEntry( Entry.FILE_ERROR_MESSAGE_FIELD, meta.getFileErrorMessageField() ) ); - result.add( getEntry( Entry.IGNORE_ERRORS, meta.isErrorIgnored() ) ); + result.add( getEntry( Entry.FILE_TYPE, meta.content.fileType) ); + result.add( getEntry( Entry.SEPARATOR, meta.content.separator ) ); + result.add( getEntry( Entry.ENCLOSURE, meta.content.enclosure ) ); + result.add( getEntry( Entry.ESCAPE_CHAR, meta.content.escapeCharacter ) ); + result.add( getEntry( Entry.BREAK_IN_ENCLOSURE, meta.content.breakInEnclosureAllowed ) ); + result.add( getEntry( Entry.HEADER_PRESENT, meta.content.header ) ); + result.add( getEntry( Entry.NR_HEADER_LINES, meta.content.nrHeaderLines ) ); + result.add( getEntry( Entry.HAS_FOOTER, meta.content.footer ) ); + result.add( getEntry( Entry.NR_FOOTER_LINES, meta.content.nrFooterLines ) ); + result.add( getEntry( Entry.HAS_WRAPPED_LINES, meta.content.lineWrapped ) ); + result.add( getEntry( Entry.NR_WRAPS, meta.content.nrWraps ) ); + result.add( getEntry( Entry.HAS_PAGED_LAYOUT, meta.content.layoutPaged ) ); + result.add( getEntry( Entry.NR_DOC_HEADER_LINES, meta.content.nrLinesDocHeader ) ); + result.add( getEntry( Entry.NR_LINES_PER_PAGE, meta.content.nrLinesPerPage ) ); + result.add( getEntry( Entry.COMPRESSION_TYPE, meta.content.fileCompression ) ); + result.add( getEntry( Entry.NO_EMPTY_LINES, meta.content.noEmptyLines ) ); + result.add( getEntry( Entry.INCLUDE_FILENAME, meta.content.includeFilename ) ); + result.add( getEntry( Entry.FILENAME_FIELD, meta.content.filenameField ) ); + result.add( getEntry( Entry.INCLUDE_ROW_NUMBER, meta.content.includeRowNumber ) ); + result.add( getEntry( Entry.ROW_NUMBER_BY_FILE, meta.content.rowNumberByFile ) ); + result.add( getEntry( Entry.ROW_NUMBER_FIELD, meta.content.rowNumberField ) ); + result.add( getEntry( Entry.FILE_FORMAT, meta.content.fileFormat ) ); + result.add( getEntry( Entry.ENCODING, meta.content.encoding ) ); + result.add( getEntry( Entry.ROW_LIMIT, meta.content.rowLimit ) ); + result.add( getEntry( Entry.DATE_FORMAT_LENIENT, meta.content.dateFormatLenient ) ); + result.add( getEntry( Entry.DATE_FORMAT_LOCALE, meta.content.dateFormatLocale ) ); + result.add( getEntry( Entry.ACCEPT_FILE_NAMES, meta.inputFiles.acceptingFilenames ) ); + result.add( getEntry( Entry.ACCEPT_FILE_STEP, meta.inputFiles.acceptingStepName ) ); + result.add( getEntry( Entry.ACCEPT_FILE_FIELD, meta.inputFiles.acceptingField ) ); + result.add( getEntry( Entry.PASS_THROUGH_FIELDS, meta.inputFiles.passingThruFields ) ); + result.add( getEntry( Entry.ADD_FILES_TO_RESULT, meta.inputFiles.isaddresult ) ); + result.add( getEntry( Entry.FILE_SHORT_FILE_FIELDNAME, meta.additionalOutputFields.shortFilenameField ) ); + result.add( getEntry( Entry.FILE_PATH_FIELDNAME, meta.additionalOutputFields.pathField ) ); + result.add( getEntry( Entry.FILE_HIDDEN_FIELDNAME, meta.additionalOutputFields.hiddenField ) ); + result.add( getEntry( Entry.FILE_LAST_MODIFICATION_FIELDNAME, meta.additionalOutputFields.lastModificationField ) ); + result.add( getEntry( Entry.FILE_URI_FIELDNAME, meta.additionalOutputFields.uriField ) ); + result.add( getEntry( Entry.FILE_EXTENSION_FIELDNAME, meta.additionalOutputFields.extensionField ) ); + result.add( getEntry( Entry.FILE_SIZE_FIELDNAME, meta.additionalOutputFields.sizeField ) ); + result.add( getEntry( Entry.SKIP_BAD_FILES, meta.errorHandling.skipBadFiles ) ); + result.add( getEntry( Entry.FILE_ERROR_FIELD, meta.errorHandling.fileErrorField ) ); + result.add( getEntry( Entry.FILE_ERROR_MESSAGE_FIELD, meta.errorHandling.fileErrorMessageField ) ); + result.add( getEntry( Entry.IGNORE_ERRORS, meta.errorHandling.errorIgnored ) ); result.add( getEntry( Entry.ERROR_COUNT_FIELD, meta.getErrorCountField() ) ); result.add( getEntry( Entry.ERROR_FIELDS_FIELD, meta.getErrorFieldsField() ) ); result.add( getEntry( Entry.ERROR_TEXT_FIELD, meta.getErrorTextField() ) ); - result.add( getEntry( Entry.WARNING_FILES_TARGET_DIR, meta.getWarningFilesDestinationDirectory() ) ); - result.add( getEntry( Entry.WARNING_FILES_EXTENTION, meta.getWarningFilesExtension() ) ); - result.add( getEntry( Entry.ERROR_FILES_TARGET_DIR, meta.getErrorFilesDestinationDirectory() ) ); - result.add( getEntry( Entry.ERROR_FILES_EXTENTION, meta.getErrorLineFilesExtension() ) ); - result.add( getEntry( Entry.LINE_NR_FILES_TARGET_DIR, meta.getLineNumberFilesDestinationDirectory() ) ); - result.add( getEntry( Entry.LINE_NR_FILES_EXTENTION, meta.getLineNumberFilesExtension() ) ); + result.add( getEntry( Entry.WARNING_FILES_TARGET_DIR, meta.errorHandling.warningFilesDestinationDirectory ) ); + result.add( getEntry( Entry.WARNING_FILES_EXTENTION, meta.errorHandling.warningFilesExtension ) ); + result.add( getEntry( Entry.ERROR_FILES_TARGET_DIR, meta.errorHandling.errorFilesDestinationDirectory ) ); + result.add( getEntry( Entry.ERROR_FILES_EXTENTION, meta.errorHandling.errorFilesExtension ) ); + result.add( getEntry( Entry.LINE_NR_FILES_TARGET_DIR, meta.errorHandling.lineNumberFilesDestinationDirectory ) ); + result.add( getEntry( Entry.LINE_NR_FILES_EXTENTION, meta.errorHandling.lineNumberFilesExtension ) ); result.add( getEntry( Entry.ERROR_LINES_SKIPPED, meta.isErrorLineSkipped() ) ); StepInjectionMetaEntry filenameLinesEntry = getEntry( Entry.FILENAME_LINES ); @@ -645,17 +644,17 @@ public List extractStepMetadataEntries() throws KettleEx List filenameLineEntryDetails = filenameLineEntry.getDetails(); filenameLineEntryDetails.add( getEntry( Entry.FILENAME, meta.getFileName()[ i ] ) ); - filenameLineEntryDetails.add( getEntry( Entry.FILEMASK, meta.getFileMask()[ i ] ) ); - filenameLineEntryDetails.add( getEntry( Entry.EXCLUDE_FILEMASK, meta.getExludeFileMask()[ i ] ) ); - filenameLineEntryDetails.add( getEntry( Entry.FILE_REQUIRED, meta.getFileRequired()[ i ] ) ); - filenameLineEntryDetails.add( getEntry( Entry.INCLUDE_SUBFOLDERS, meta.getIncludeSubFolders()[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.FILEMASK, meta.inputFiles.fileMask[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.EXCLUDE_FILEMASK, meta.inputFiles.excludeFileMask[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.FILE_REQUIRED, meta.inputFiles.fileRequired[ i ] ) ); + filenameLineEntryDetails.add( getEntry( Entry.INCLUDE_SUBFOLDERS, meta.inputFiles.includeSubFolders[ i ] ) ); } } result.add( filenameLinesEntry ); StepInjectionMetaEntry fieldsEntry = getEntry( Entry.FIELDS ); - if ( !Const.isEmpty( meta.getInputFields() ) ) { - for ( TextFileInputField inputField : meta.getInputFields() ) { + if ( !Const.isEmpty( meta.inputFiles.inputFields ) ) { + for ( TextFileInputField inputField : meta.inputFiles.inputFields ) { StepInjectionMetaEntry fieldEntry = getEntry( Entry.FIELD ); fieldsEntry.getDetails().add( fieldEntry ); diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputReader.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputReader.java new file mode 100644 index 000000000000..a4da55468b10 --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputReader.java @@ -0,0 +1,466 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.textfileinput; + +import java.io.BufferedInputStream; +import java.io.InputStreamReader; + +import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.compress.CompressionInputStream; +import org.pentaho.di.core.compress.CompressionProvider; +import org.pentaho.di.core.compress.CompressionProviderFactory; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.trans.step.BaseStep; +import org.pentaho.di.trans.step.errorhandling.AbstractFileErrorHandler; +import org.pentaho.di.trans.steps.baseinput.IBaseInputReader; +import org.pentaho.di.trans.steps.baseinput.IBaseInputStepControl; + +/** + * Reader for one text file. + * + * @author Alexander Buloichik + */ +public class TextFileInputReader implements IBaseInputReader { + private static final int BUFFER_SIZE_INPUT_STREAM = 500; + + private final IBaseInputStepControl step; + private final TextFileInputMeta meta; + private final TextFileInputData data; + private final LogChannelInterface log; + + private final CompressionInputStream in; + + private final InputStreamReader isr; + + protected long lineInFile; + + private boolean first; + + protected long lineNumberInFile; + + public TextFileInputReader( IBaseInputStepControl step, TextFileInputMeta meta, TextFileInputData data, FileObject file, + LogChannelInterface log ) throws Exception { + this.step = step; + this.meta = meta; + this.data = data; + this.log = log; + + CompressionProvider provider = + CompressionProviderFactory.getInstance().getCompressionProviderByName( meta.content.fileCompression ); + + if ( log.isDetailed() ) { + log.logDetailed( "This is a compressed file being handled by the " + provider.getName() + " provider" ); + } + + in = provider.createInputStream( KettleVFS.getInputStream( file ) ); + + in.nextEntry(); + + if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { + isr = new InputStreamReader( new BufferedInputStream( in, BUFFER_SIZE_INPUT_STREAM ), meta.getEncoding() ); + } else { + isr = new InputStreamReader( new BufferedInputStream( in, BUFFER_SIZE_INPUT_STREAM ) ); + } + + String encoding = isr.getEncoding(); + data.encodingType = EncodingType.guessEncodingType( encoding ); + + readInitial(); + } + + protected void readInitial() throws Exception { + data.doneWithHeader = !meta.content.header; + // ///////////////////////////////////////////////////////////////////////////// + // Read the first lines... + + /* + * Keep track of the status of the file: are there any lines left to read? + */ + data.doneReading = false; + + /* + * OK, read a number of lines in the buffer: The header rows The nr rows in the page : optional The footer rows + */ + int bufferSize = 1; + bufferSize += meta.content.header ? meta.content.nrHeaderLines : 0; + bufferSize += + meta.content.layoutPaged ? meta.content.nrLinesPerPage * ( Math.max( 0, meta.content.nrWraps ) + 1 ) : Math.max( 0, meta + .content.nrWraps ); // it helps when we have wrapped input w/o header + + bufferSize += meta.content.footer ? meta.content.nrFooterLines : 0; + + // See if we need to skip the document header lines... + if ( meta.content.layoutPaged ) { + for ( int i = 0; i < meta.content.nrLinesDocHeader; i++ ) { + // Just skip these... + TextFileInputUtils.getLine( log, isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); // header + // and + // footer: not + // wrapped + lineNumberInFile++; + } + } + + for ( int i = 0; i < bufferSize && !data.doneReading; i++ ) { + boolean wasNotFiltered = tryToReadLine( !meta.content.header || i >= meta.content.nrHeaderLines ); + if ( !wasNotFiltered ) { + // grab another line, this one got filtered + bufferSize++; + } + } + + // Reset counters etc. + data.headerLinesRead = 0; + data.footerLinesRead = 0; + data.pageLinesRead = 0; + + } + + @Override + public boolean readRow() throws KettleException { + Object[] r = null; + boolean retval = true; + boolean putrow = false; + + if ( !data.doneReading ) { + int repeats = 1; + if ( meta.content.lineWrapped ) { + repeats = meta.content.nrWraps > 0 ? meta.content.nrWraps : repeats; + } + + if ( !data.doneWithHeader && data.headerLinesRead == 0 ) { + // We are just starting to read header lines, read them all + repeats += meta.content.nrHeaderLines + 1; + } + + // Read a number of lines... + for ( int i = 0; i < repeats && !data.doneReading; i++ ) { + tryToReadLine( true ); + } + } + + if ( data.lineBuffer.isEmpty() ) { + return false; + } + + /* + * Take the first line available in the buffer & remove the line from the buffer + */ + TextFileLine textLine = data.lineBuffer.get( 0 ); + step.incrementLinesInput(); + lineNumberInFile++; + + data.lineBuffer.remove( 0 ); + + if ( meta.content.layoutPaged ) { + /* + * Different rules apply: on each page: a header a number of data lines a footer + */ + if ( !data.doneWithHeader && data.pageLinesRead == 0 ) // We are reading header lines + { + if ( log.isRowLevel() ) { + log.logRowlevel( "P-HEADER (" + data.headerLinesRead + ") : " + textLine.line ); + } + data.headerLinesRead++; + if ( data.headerLinesRead >= meta.content.nrHeaderLines ) { + data.doneWithHeader = true; + } + } else { + // data lines or footer on a page + + if ( data.pageLinesRead < meta.content.nrLinesPerPage ) { + // See if we are dealing with wrapped lines: + if ( meta.content.lineWrapped ) { + for ( int i = 0; i < meta.content.nrWraps; i++ ) { + String extra = ""; + if ( data.lineBuffer.size() > 0 ) { + extra = data.lineBuffer.get( 0 ).line; + data.lineBuffer.remove( 0 ); + } + textLine.line += extra; + } + } + + if ( log.isRowLevel() ) { + log.logRowlevel( "P-DATA: " + textLine.line ); + } + // Read a normal line on a page of data. + data.pageLinesRead++; + lineInFile++; + long useNumber = meta.content.rowNumberByFile ? lineInFile : step.getLinesWritten() + 1; + r = + TextFileInputUtils.convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, + data.nrPassThruFields, data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, + data.separator, data.enclosure, data.escapeCharacter, data.dataErrorLineHandler, + meta.additionalOutputFields, data.shortFilename, data.path, data.hidden, + data.lastModificationDateTime, data.uriName, data.rootUriName, data.extension, data.size ); + if ( r != null ) { + putrow = true; + } + + // Possible fix for bug PDI-1121 - paged layout header and line count off by 1 + // We need to reset these BEFORE the next header line is read, so that it + // is treated as a header ... obviously, only if there is no footer, and we are + // done reading data. + if ( !meta.content.footer && ( data.pageLinesRead == meta.content.nrLinesPerPage ) ) { + /* + * OK, we are done reading the footer lines, start again on 'next page' with the header + */ + data.doneWithHeader = false; + data.headerLinesRead = 0; + data.pageLinesRead = 0; + data.footerLinesRead = 0; + if ( log.isRowLevel() ) { + log.logRowlevel( "RESTART PAGE" ); + } + } + } else { + // done reading the data lines, skip the footer lines + + if ( meta.content.footer && data.footerLinesRead < meta.content.nrFooterLines ) { + if ( log.isRowLevel() ) { + log.logRowlevel( "P-FOOTER: " + textLine.line ); + } + data.footerLinesRead++; + } + + if ( !meta.content.footer || data.footerLinesRead >= meta.content.nrFooterLines ) { + /* + * OK, we are done reading the footer lines, start again on 'next page' with the header + */ + data.doneWithHeader = false; + data.headerLinesRead = 0; + data.pageLinesRead = 0; + data.footerLinesRead = 0; + if ( log.isRowLevel() ) { + log.logRowlevel( "RESTART PAGE" ); + } + } + } + } + } else { + // A normal data line, can also be a header or a footer line + + if ( !data.doneWithHeader ) { // We are reading header lines + + data.headerLinesRead++; + if ( data.headerLinesRead >= meta.content.nrHeaderLines ) { + data.doneWithHeader = true; + } + } else { + /* + * IF we are done reading and we have a footer AND the number of lines in the buffer is smaller then the number + * of footer lines THEN we can remove the remaining rows from the buffer: they are all footer rows. + */ + if ( data.doneReading && meta.content.footer && data.lineBuffer.size() < meta.content.nrFooterLines ) { + data.lineBuffer.clear(); + } else { + // Not yet a footer line: it's a normal data line. + + // See if we are dealing with wrapped lines: + if ( meta.content.lineWrapped ) { + for ( int i = 0; i < meta.content.nrWraps; i++ ) { + String extra = ""; + if ( data.lineBuffer.size() > 0 ) { + extra = data.lineBuffer.get( 0 ).line; + data.lineBuffer.remove( 0 ); + } else { + tryToReadLine( true ); + if ( !data.lineBuffer.isEmpty() ) { + extra = data.lineBuffer.remove( 0 ).line; + } + } + textLine.line += extra; + } + } + if ( data.filePlayList.isProcessingNeeded( textLine.file, textLine.lineNumber, + AbstractFileErrorHandler.NO_PARTS ) ) { + lineInFile++; + long useNumber = meta.content.rowNumberByFile ? lineInFile : step.getLinesWritten() + 1; + r = + TextFileInputUtils.convertLineToRow( log, textLine, meta, data.currentPassThruFieldsRow, + data.nrPassThruFields, data.outputRowMeta, data.convertRowMeta, data.filename, useNumber, + data.separator, data.enclosure, data.escapeCharacter, data.dataErrorLineHandler, + meta.additionalOutputFields, data.shortFilename, data.path, data.hidden, + data.lastModificationDateTime, data.uriName, data.rootUriName, data.extension, data.size ); + if ( r != null ) { + if ( log.isRowLevel() ) { + log.logRowlevel( "Found data row: " + data.outputRowMeta.getString( r ) ); + } + putrow = true; + } + } else { + putrow = false; + } + } + } + } + + if ( putrow && r != null ) { + // See if the previous values need to be repeated! + if ( data.nr_repeats > 0 ) { + if ( data.previous_row == null ) { // First invocation... + + data.previous_row = data.outputRowMeta.cloneRow( r ); + } else { + // int repnr = 0; + for ( int i = 0; i < meta.inputFiles.inputFields.length; i++ ) { + if ( meta.inputFiles.inputFields[i].isRepeated() ) { + if ( r[i] == null ) { + // if it is empty: take the previous value! + + r[i] = data.previous_row[i]; + } else { + // not empty: change the previous_row entry! + + data.previous_row[i] = r[i]; + } + // repnr++; + } + } + } + } + + if ( log.isRowLevel() ) { + log.logRowlevel( "Putting row: " + data.outputRowMeta.getString( r ) ); + } + step.putRow( data.outputRowMeta, r ); + + if ( step.getLinesInput() >= meta.content.rowLimit && meta.content.rowLimit > 0 ) { + close(); + return false; + } + } + + if ( step.checkFeedback( step.getLinesInput() ) ) { + if ( log.isBasic() ) { + log.logBasic( "linenr " + step.getLinesInput() ); + } + } + + return retval; + } + + @Override + public void close() { + try { + // Close previous file! + if ( data.filename != null ) { + // Increment the lines updated to reflect another file has been finished. + // This allows us to give a state of progress in the run time metrics + step.incrementLinesUpdated(); + /* + * } else if ( sFileCompression != null && sFileCompression.equals( "Snappy" ) && data.sis != null ) { + * data.sis.close(); } + */ + if ( in != null ) { + BaseStep.closeQuietly( in ); + } + isr.close(); + data.filename = null; // send it down the next time. + if ( data.file != null ) { + try { + data.file.close(); + data.file = null; + } catch ( Exception e ) { + log.logError( "Error closing file", e ); + } + data.file = null; + } + } + data.dataErrorLineHandler.close(); + } catch ( Exception e ) { + String errorMsg = "Couldn't close file : " + data.file.getName().getFriendlyURI() + " --> " + e.toString(); + log.logError( errorMsg ); + if ( step.failAfterBadFile( errorMsg ) ) { // ( !meta.isSkipBadFiles() || data.isLastFile ){ + step.stopAll(); + } + step.setErrors( step.getErrors() + 1 ); + } // finally { + // This is for bug #5797 : it tries to assure that the file handle + // is actually freed/garbarge collected. + // XXX deinspanjer 2009-07-07: I'm stubbing this out. The bug was ancient and it is worth reevaluating + // to avoid the performance hit of a System GC on every file close + // System.gc(); + // } + } + + protected boolean tryToReadLine( boolean applyFilter ) throws KettleFileException { + String line; + line = TextFileInputUtils.getLine( log, isr, data.encodingType, data.fileFormatType, data.lineStringBuilder ); + if ( line != null ) { + // when there is no header, check the filter for the first line + if ( applyFilter ) { + // Filter row? + boolean isFilterLastLine = false; + boolean filterOK = checkFilterRow( line, isFilterLastLine ); + if ( filterOK ) { + data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the + // line buffer... + } else { + return false; + } + } else { // don't checkFilterRow + + if ( !meta.content.noEmptyLines || line.length() != 0 ) { + data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the line + // buffer... + } + } + } else { + data.doneReading = true; + } + return true; + } + + /** + * Check if the line should be taken. + * + * @param line + * @param isFilterLastLine + * (dummy input param, only set when return value is false) + * @return true when the line should be taken (when false, isFilterLastLine will be set) + */ + private boolean checkFilterRow( String line, boolean isFilterLastLine ) { + boolean filterOK = true; + + // check for noEmptyLines + if ( meta.content.noEmptyLines && line.length() == 0 ) { + filterOK = false; + } else { + // check the filters + filterOK = data.filterProcessor.doFilters( line ); + if ( !filterOK ) { + if ( data.filterProcessor.isStopProcessing() ) { + data.doneReading = true; + } + } + } + + return filterOK; + } + +} diff --git a/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputUtils.java b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputUtils.java new file mode 100644 index 000000000000..6fd69a849ffa --- /dev/null +++ b/engine/src/org/pentaho/di/trans/steps/textfileinput/TextFileInputUtils.java @@ -0,0 +1,758 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.textfileinput; + +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.pentaho.di.core.Const; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.row.RowDataUtil; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.step.errorhandling.AbstractFileErrorHandler; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.steps.baseinput.BaseInputStepMeta; + +/** + * Some common methods for text file parsing. + * + * @author Alexander Buloichik + */ +public class TextFileInputUtils { + private static Class PKG = TextFileInputUtils.class; // for i18n purposes, needed by Translator2!! + + public static final String[] guessStringsFromLine( VariableSpace space, LogChannelInterface log, String line, + TextFileInputMeta inf, String delimiter, String enclosure, String escapeCharacter ) throws KettleException { + List strings = new ArrayList(); + + String pol; // piece of line + + try { + if ( line == null ) { + return null; + } + + if ( inf.content.fileType.equalsIgnoreCase( "CSV" ) ) { + + // Split string in pieces, only for CSV! + + int pos = 0; + int length = line.length(); + boolean dencl = false; + + int len_encl = ( enclosure == null ? 0 : enclosure.length() ); + int len_esc = ( escapeCharacter == null ? 0 : escapeCharacter.length() ); + + while ( pos < length ) { + int from = pos; + int next; + + boolean encl_found; + boolean contains_escaped_enclosures = false; + boolean contains_escaped_separators = false; + + // Is the field beginning with an enclosure? + // "aa;aa";123;"aaa-aaa";000;... + if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.ConvertLineToRow", line.substring( from, from + len_encl ) ) ); + } + encl_found = true; + int p = from + len_encl; + + boolean is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equalsIgnoreCase( + enclosure ); + boolean is_escape = + len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equalsIgnoreCase( + escapeCharacter ); + + boolean enclosure_after = false; + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equalsIgnoreCase( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; + } + } + } + + // Look for a closing enclosure! + while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { + p++; + enclosure_after = false; + is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equals( enclosure ); + is_escape = + len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equals( escapeCharacter ); + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equals( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; // remember + } + } + } + } + + if ( p >= length ) { + next = p; + } else { + next = p + len_encl; + } + + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); + } + } else { + encl_found = false; + boolean found = false; + int startpoint = from; + // int tries = 1; + do { + next = line.indexOf( delimiter, startpoint ); + + // See if this position is preceded by an escape character. + if ( len_esc > 0 && next - len_esc > 0 ) { + String before = line.substring( next - len_esc, next ); + + if ( escapeCharacter.equals( before ) ) { + // take the next separator, this one is escaped... + startpoint = next + 1; + // tries++; + contains_escaped_separators = true; + } else { + found = true; + } + } else { + found = true; + } + } while ( !found && next >= 0 ); + } + if ( next == -1 ) { + next = length; + } + + if ( encl_found ) { + pol = line.substring( from + len_encl, next - len_encl ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); + } + } else { + pol = line.substring( from, next ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); + } + } + + if ( dencl ) { + StringBuilder sbpol = new StringBuilder( pol ); + int idx = sbpol.indexOf( enclosure + enclosure ); + while ( idx >= 0 ) { + sbpol.delete( idx, idx + enclosure.length() ); + idx = sbpol.indexOf( enclosure + enclosure ); + } + pol = sbpol.toString(); + } + + // replace the escaped enclosures with enclosures... + if ( contains_escaped_enclosures ) { + String replace = escapeCharacter + enclosure; + String replaceWith = enclosure; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // replace the escaped separators with separators... + if ( contains_escaped_separators ) { + String replace = escapeCharacter + delimiter; + String replaceWith = delimiter; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // Now add pol to the strings found! + strings.add( pol ); + + pos = next + delimiter.length(); + } + if ( pos == length ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); + } + strings.add( "" ); + } + } else { + // Fixed file format: Simply get the strings at the required positions... + for ( int i = 0; i < inf.inputFiles.inputFields.length; i++ ) { + TextFileInputField field = inf.inputFiles.inputFields[i]; + + int length = line.length(); + + if ( field.getPosition() + field.getLength() <= length ) { + strings.add( line.substring( field.getPosition(), field.getPosition() + field.getLength() ) ); + } else { + if ( field.getPosition() < length ) { + strings.add( line.substring( field.getPosition() ) ); + } else { + strings.add( "" ); + } + } + } + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e + .toString() ), e ); + } + + return strings.toArray( new String[strings.size()] ); + } + + public static final String getLine( LogChannelInterface log, InputStreamReader reader, int formatNr, + StringBuilder line ) throws KettleFileException { + EncodingType type = EncodingType.guessEncodingType( reader.getEncoding() ); + return getLine( log, reader, type, formatNr, line ); + } + + public static final String getLine( LogChannelInterface log, InputStreamReader reader, EncodingType encodingType, + int formatNr, StringBuilder line ) throws KettleFileException { + int c = 0; + line.setLength( 0 ); + try { + switch ( formatNr ) { + case TextFileInputMeta.FILE_FORMAT_DOS: + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isReturn( c ) || encodingType.isLinefeed( c ) ) { + c = reader.read(); // skip \n and \r + if ( !encodingType.isReturn( c ) && !encodingType.isLinefeed( c ) ) { + // make sure its really a linefeed or cariage return + // raise an error this is not a DOS file + // so we have pulled a character from the next line + throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.SingleLineFound" ) ); + } + return line.toString(); + } + if ( c >= 0 ) { + line.append( (char) c ); + } + } + break; + case TextFileInputMeta.FILE_FORMAT_UNIX: + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isLinefeed( c ) || encodingType.isReturn( c ) ) { + return line.toString(); + } + if ( c >= 0 ) { + line.append( (char) c ); + } + } + break; + case TextFileInputMeta.FILE_FORMAT_MIXED: + // in mixed mode we suppose the LF is the last char and CR is ignored + // not for MAC OS 9 but works for Mac OS X. Mac OS 9 can use UNIX-Format + while ( c >= 0 ) { + c = reader.read(); + + if ( encodingType.isLinefeed( c ) ) { + return line.toString(); + } else if ( !encodingType.isReturn( c ) ) { + if ( c >= 0 ) { + line.append( (char) c ); + } + } + } + break; + default: + break; + } + } catch ( KettleFileException e ) { + throw e; + } catch ( Exception e ) { + if ( line.length() == 0 ) { + throw new KettleFileException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ExceptionReadingLine", e + .toString() ), e ); + } + return line.toString(); + } + if ( line.length() > 0 ) { + return line.toString(); + } + + return null; + } + + public static final Object[] convertLineToRow( LogChannelInterface log, TextFileLine NewFileLine, + TextFileInputMeta info, Object[] passThruFields, int nrPassThruFields, RowMetaInterface outputRowMeta, + RowMetaInterface convertRowMeta, String fname, long rowNr, String delimiter, String enclosure, + String escapeCharacter, FileErrorHandler errorHandler, + BaseInputStepMeta.AdditionalOutputFields additionalOutputFIelds, String shortFilename, String path, + boolean hidden, Date modificationDateTime, String uri, String rooturi, String extension, long size ) + throws KettleException { + if ( NewFileLine == null || NewFileLine.line == null ) { + return null; + } + + Object[] r = RowDataUtil.allocateRowData( outputRowMeta.size() ); // over-allocate a bit in the row producing + // steps... + + int nrfields = info.inputFiles.inputFields.length; + int fieldnr; + + Long errorCount = null; + if ( info.errorHandling.errorIgnored && info.getErrorCountField() != null && info.getErrorCountField() + .length() > 0 ) { + errorCount = new Long( 0L ); + } + String errorFields = null; + if ( info.errorHandling.errorIgnored && info.getErrorFieldsField() != null && info.getErrorFieldsField() + .length() > 0 ) { + errorFields = ""; + } + String errorText = null; + if ( info.errorHandling.errorIgnored && info.getErrorTextField() != null && info.getErrorTextField() + .length() > 0 ) { + errorText = ""; + } + + try { + // System.out.println("Convertings line to string ["+line+"]"); + String[] strings = convertLineToStrings( log, NewFileLine.line, info, delimiter, enclosure, escapeCharacter ); + int shiftFields = ( passThruFields == null ? 0 : nrPassThruFields ); + for ( fieldnr = 0; fieldnr < nrfields; fieldnr++ ) { + TextFileInputField f = info.inputFiles.inputFields[fieldnr]; + int valuenr = shiftFields + fieldnr; + ValueMetaInterface valueMeta = outputRowMeta.getValueMeta( valuenr ); + ValueMetaInterface convertMeta = convertRowMeta.getValueMeta( valuenr ); + + Object value; + + String nullif = fieldnr < nrfields ? f.getNullString() : ""; + String ifnull = fieldnr < nrfields ? f.getIfNullValue() : ""; + int trim_type = fieldnr < nrfields ? f.getTrimType() : ValueMetaInterface.TRIM_TYPE_NONE; + + if ( fieldnr < strings.length ) { + String pol = strings[fieldnr]; + try { + value = valueMeta.convertDataFromString( pol, convertMeta, nullif, ifnull, trim_type ); + } catch ( Exception e ) { + // OK, give some feedback! + String message = + BaseMessages.getString( PKG, "TextFileInput.Log.CoundNotParseField", valueMeta.toStringMeta(), "" + pol, + valueMeta.getConversionMask(), "" + rowNr ); + + if ( info.errorHandling.errorIgnored ) { + log.logDetailed( fname, BaseMessages.getString( PKG, "TextFileInput.Log.Warning" ) + ": " + message + + " : " + e.getMessage() ); + + value = null; + + if ( errorCount != null ) { + errorCount = new Long( errorCount.longValue() + 1L ); + } + if ( errorFields != null ) { + StringBuilder sb = new StringBuilder( errorFields ); + if ( sb.length() > 0 ) { + sb.append( "\t" ); // TODO document this change + } + sb.append( valueMeta.getName() ); + errorFields = sb.toString(); + } + if ( errorText != null ) { + StringBuilder sb = new StringBuilder( errorText ); + if ( sb.length() > 0 ) { + sb.append( Const.CR ); + } + sb.append( message ); + errorText = sb.toString(); + } + if ( errorHandler != null ) { + errorHandler.handleLineError( NewFileLine.lineNumber, AbstractFileErrorHandler.NO_PARTS ); + } + + if ( info.isErrorLineSkipped() ) { + r = null; // compensates for stmt: r.setIgnore(); + } + } else { + throw new KettleException( message, e ); + } + } + } else { + // No data found: TRAILING NULLCOLS: add null value... + value = null; + } + + // Now add value to the row (if we're not skipping the row) + if ( r != null ) { + r[valuenr] = value; + } + } + + // none of this applies if we're skipping the row + if ( r != null ) { + // Support for trailing nullcols! + // Should be OK at allocation time, but it doesn't hurt :-) + if ( fieldnr < nrfields ) { + for ( int i = fieldnr; i < info.inputFiles.inputFields.length; i++ ) { + r[shiftFields + i] = null; + } + } + + // Add the error handling fields... + int index = shiftFields + nrfields; + if ( errorCount != null ) { + r[index] = errorCount; + index++; + } + if ( errorFields != null ) { + r[index] = errorFields; + index++; + } + if ( errorText != null ) { + r[index] = errorText; + index++; + } + + // Possibly add a filename... + if ( info.content.includeFilename ) { + r[index] = fname; + index++; + } + + // Possibly add a row number... + if ( info.content.includeRowNumber ) { + r[index] = new Long( rowNr ); + index++; + } + + // Possibly add short filename... + if ( additionalOutputFIelds.shortFilenameField != null ) { + r[index] = shortFilename; + index++; + } + // Add Extension + if ( additionalOutputFIelds.extensionField != null ) { + r[index] = extension; + index++; + } + // add path + if ( additionalOutputFIelds.pathField != null ) { + r[index] = path; + index++; + } + // Add Size + if ( additionalOutputFIelds.sizeField != null ) { + r[index] = new Long( size ); + index++; + } + // add Hidden + if ( additionalOutputFIelds.hiddenField != null ) { + r[index] = hidden; + index++; + } + // Add modification date + if ( additionalOutputFIelds.lastModificationField != null ) { + r[index] = modificationDateTime; + index++; + } + // Add Uri + if ( additionalOutputFIelds.uriField != null ) { + r[index] = uri; + index++; + } + // Add RootUri + if ( additionalOutputFIelds.rootUriField != null ) { + r[index] = rooturi; + index++; + } + } // End if r != null + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLineText" ), e ); + } + + if ( passThruFields != null ) { + // Simply add all fields from source files step + for ( int i = 0; i < nrPassThruFields; i++ ) { + r[i] = passThruFields[i]; + } + } + + return r; + + } + + public static final String[] convertLineToStrings( LogChannelInterface log, String line, TextFileInputMeta inf, + String delimiter, String enclosure, String escapeCharacters ) throws KettleException { + String[] strings = new String[inf.inputFiles.inputFields.length]; + int fieldnr; + + String pol; // piece of line + + try { + if ( line == null ) { + return null; + } + + if ( inf.content.fileType.equalsIgnoreCase( "CSV" ) ) { + // Split string in pieces, only for CSV! + + fieldnr = 0; + int pos = 0; + int length = line.length(); + boolean dencl = false; + + int len_encl = ( enclosure == null ? 0 : enclosure.length() ); + int len_esc = ( escapeCharacters == null ? 0 : escapeCharacters.length() ); + + while ( pos < length ) { + int from = pos; + int next; + + boolean encl_found; + boolean contains_escaped_enclosures = false; + boolean contains_escaped_separators = false; + + // Is the field beginning with an enclosure? + // "aa;aa";123;"aaa-aaa";000;... + if ( len_encl > 0 && line.substring( from, from + len_encl ).equalsIgnoreCase( enclosure ) ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.Encloruse", line.substring( from, from + len_encl ) ) ); + } + encl_found = true; + int p = from + len_encl; + + boolean is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equalsIgnoreCase( + enclosure ); + boolean is_escape = + len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equalsIgnoreCase( inf + .content.escapeCharacter ); + + boolean enclosure_after = false; + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equalsIgnoreCase( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; + } + } + } + + // Look for a closing enclosure! + while ( ( !is_enclosure || enclosure_after ) && p < line.length() ) { + p++; + enclosure_after = false; + is_enclosure = + len_encl > 0 && p + len_encl < length && line.substring( p, p + len_encl ).equals( enclosure ); + is_escape = + len_esc > 0 && p + len_esc < length && line.substring( p, p + len_esc ).equals( inf + .content.escapeCharacter ); + + // Is it really an enclosure? See if it's not repeated twice or escaped! + if ( ( is_enclosure || is_escape ) && p < length - 1 ) { + + String strnext = line.substring( p + len_encl, p + 2 * len_encl ); + if ( strnext.equals( enclosure ) ) { + p++; + enclosure_after = true; + dencl = true; + + // Remember to replace them later on! + if ( is_escape ) { + contains_escaped_enclosures = true; // remember + } + } + } + } + + if ( p >= length ) { + next = p; + } else { + next = p + len_encl; + } + + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEnclosure", "" + p ) ); + } + } else { + encl_found = false; + boolean found = false; + int startpoint = from; + // int tries = 1; + do { + next = line.indexOf( delimiter, startpoint ); + + // See if this position is preceded by an escape character. + if ( len_esc > 0 && next - len_esc > 0 ) { + String before = line.substring( next - len_esc, next ); + + if ( inf.content.escapeCharacter.equals( before ) ) { + // take the next separator, this one is escaped... + startpoint = next + 1; + // tries++; + contains_escaped_separators = true; + } else { + found = true; + } + } else { + found = true; + } + } while ( !found && next >= 0 ); + } + if ( next == -1 ) { + next = length; + } + + if ( encl_found && ( ( from + len_encl ) <= ( next - len_encl ) ) ) { + pol = line.substring( from + len_encl, next - len_encl ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EnclosureFieldFound", "" + pol ) ); + } + } else { + pol = line.substring( from, next ); + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.NormalFieldFound", "" + pol ) ); + } + } + + if ( dencl && Const.isEmpty( inf.content.escapeCharacter ) ) { + StringBuilder sbpol = new StringBuilder( pol ); + int idx = sbpol.indexOf( enclosure + enclosure ); + while ( idx >= 0 ) { + sbpol.delete( idx, idx + enclosure.length() ); + idx = sbpol.indexOf( enclosure + enclosure ); + } + pol = sbpol.toString(); + } + + // replace the escaped enclosures with enclosures... + if ( contains_escaped_enclosures ) { + String replace = inf.content.escapeCharacter + enclosure; + String replaceWith = enclosure; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // replace the escaped separators with separators... + if ( contains_escaped_separators ) { + String replace = inf.content.escapeCharacter + delimiter; + String replaceWith = delimiter; + + pol = Const.replace( pol, replace, replaceWith ); + } + + // Now add pol to the strings found! + try { + strings[fieldnr] = pol; + } catch ( ArrayIndexOutOfBoundsException e ) { + // In case we didn't allocate enough space. + // This happens when you have less header values specified than there are actual values in the rows. + // As this is "the exception" we catch and resize here. + // + String[] newStrings = new String[strings.length]; + for ( int x = 0; x < strings.length; x++ ) { + newStrings[x] = strings[x]; + } + strings = newStrings; + } + + pos = next + delimiter.length(); + fieldnr++; + } + if ( pos == length ) { + if ( log.isRowLevel() ) { + log.logRowlevel( BaseMessages.getString( PKG, "TextFileInput.Log.ConvertLineToRowTitle" ), BaseMessages + .getString( PKG, "TextFileInput.Log.EndOfEmptyLineFound" ) ); + } + if ( fieldnr < strings.length ) { + strings[fieldnr] = Const.EMPTY_STRING; + } + fieldnr++; + } + } else { + // Fixed file format: Simply get the strings at the required positions... + for ( int i = 0; i < inf.inputFiles.inputFields.length; i++ ) { + TextFileInputField field = inf.inputFiles.inputFields[i]; + + int length = line.length(); + + if ( field.getPosition() + field.getLength() <= length ) { + strings[i] = line.substring( field.getPosition(), field.getPosition() + field.getLength() ); + } else { + if ( field.getPosition() < length ) { + strings[i] = line.substring( field.getPosition() ); + } else { + strings[i] = ""; + } + } + } + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInput.Log.Error.ErrorConvertingLine", e + .toString() ), e ); + } + + return strings; + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/EncodingTypeTest.java b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/EncodingTypeTest.java new file mode 100644 index 000000000000..b24189f0da67 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/EncodingTypeTest.java @@ -0,0 +1,63 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * User: Dzmitry Stsiapanau Date: 3/11/14 Time: 11:44 AM + */ +public class EncodingTypeTest { + @Test + public void testIsReturn() throws Exception { + int lineFeed = '\n'; + int carriageReturn = '\r'; + assertTrue( "SINGLE.isLineFeed is not line feed", OldEncodingType.SINGLE.isLinefeed( lineFeed ) ); + assertTrue( "DOUBLE_BIG_ENDIAN is not line feed", OldEncodingType.DOUBLE_BIG_ENDIAN.isLinefeed( lineFeed ) ); + assertTrue( "DOUBLE_LITTLE_ENDIAN.isLineFeed is not line feed", + OldEncodingType.DOUBLE_LITTLE_ENDIAN.isLinefeed( lineFeed ) ); + assertFalse( "SINGLE.isLineFeed is carriage return", OldEncodingType.SINGLE.isLinefeed( carriageReturn ) ); + assertFalse( "DOUBLE_BIG_ENDIAN.isLineFeed is carriage return", + OldEncodingType.DOUBLE_BIG_ENDIAN.isLinefeed( carriageReturn ) ); + assertFalse( "DOUBLE_LITTLE_ENDIAN.isLineFeed is carriage return", + OldEncodingType.DOUBLE_LITTLE_ENDIAN.isLinefeed( carriageReturn ) ); + } + + @Test + public void testIsLinefeed() throws Exception { + int lineFeed = '\n'; + int carriageReturn = '\r'; + assertFalse( "SINGLE.isReturn is line feed", OldEncodingType.SINGLE.isReturn( lineFeed ) ); + assertFalse( "DOUBLE_BIG_ENDIAN.isReturn is line feed", OldEncodingType.DOUBLE_BIG_ENDIAN.isReturn( lineFeed ) ); + assertFalse( "DOUBLE_LITTLE_ENDIAN.isReturn is line feed", + OldEncodingType.DOUBLE_LITTLE_ENDIAN.isReturn( lineFeed ) ); + assertTrue( "SINGLE.isReturn is not carriage return", OldEncodingType.SINGLE.isReturn( carriageReturn ) ); + assertTrue( "DOUBLE_BIG_ENDIAN.isReturn is not carriage return", + OldEncodingType.DOUBLE_BIG_ENDIAN.isReturn( carriageReturn ) ); + assertTrue( "DOUBLE_LITTLE_ENDIAN.isReturn is not carriage return", + OldEncodingType.DOUBLE_LITTLE_ENDIAN.isReturn( carriageReturn ) ); + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjectionTest.java b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjectionTest.java new file mode 100644 index 000000000000..b996e056df22 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaInjectionTest.java @@ -0,0 +1,184 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import org.junit.Test; +import org.pentaho.di.trans.step.StepInjectionMetaEntry; +import org.pentaho.di.trans.step.StepInjectionUtil; +import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; +import org.pentaho.di.trans.steps.loadsave.validator.IntLoadSaveValidator; +import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static java.util.Arrays.asList; +import static org.junit.Assert.*; +import static org.pentaho.di.core.row.ValueMetaInterface.TYPE_NONE; +import static org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileInputMetaInjection.Entry.*; + +public class OldTextFileInputMetaInjectionTest { + + @Test + public void extractingAll() throws Exception { + OldTextFileInputMetaInjection injection = new OldTextFileInputMetaInjection( new OldTextFileInputMeta() ); + List metadata = injection.getStepInjectionMetadataEntries(); + List extracted = injection.extractStepMetadataEntries(); + + assertEquals( metadata.size(), extracted.size() ); + for ( StepInjectionMetaEntry metaEntry : metadata ) { + assertNotNull( metaEntry.getKey(), StepInjectionUtil.findEntry( extracted, metaEntry.getKey() ) ); + } + } + + + @Test + public void topEntriesAreInjected() throws Exception { + OldTextFileInputMetaInjection.Entry[] topEntries = OldTextFileInputMetaInjection.Entry.getTopEntries(); + List injectionValues = createInjectionValues( topEntries ); + + OldTextFileInputMetaInjection injection = new OldTextFileInputMetaInjection( new OldTextFileInputMeta() ); + injection.injectStepMetadataEntries( injectionValues ); + + assertInjected( injection.extractStepMetadataEntries(), injectionValues ); + } + + private static List createInjectionValues( OldTextFileInputMetaInjection.Entry[] entries ) { + Map> generators = createGeneratorsMapping(); + List result = new ArrayList( entries.length ); + for ( OldTextFileInputMetaInjection.Entry entry : entries ) { + StepInjectionMetaEntry injectionEntry = StepInjectionUtil.getEntry( entry ); + if ( entry.getValueType() != TYPE_NONE ) { + injectionEntry.setValue( generators.get( entry ).generateValue() ); + } + result.add( injectionEntry ); + } + return result; + } + + private static Map> createGeneratorsMapping() { + Map> generators = new EnumMap>( OldTextFileInputMetaInjection.Entry.class ); + + Generator stringGenerator = new ValidatorAdapter( new StringLoadSaveValidator() ); + List stringEntries = asList( + FILE_TYPE, SEPARATOR, ENCLOSURE, ESCAPE_CHAR, COMPRESSION_TYPE, FILENAME_FIELD, ROW_NUMBER_FIELD, + FILE_FORMAT, ENCODING, ACCEPT_FILE_STEP, ACCEPT_FILE_FIELD, FILE_SHORT_FILE_FIELDNAME, FILE_PATH_FIELDNAME, + FILE_LAST_MODIFICATION_FIELDNAME, FILE_URI_FIELDNAME, FILE_EXTENSION_FIELDNAME, FILE_SIZE_FIELDNAME, + FILE_ERROR_FIELD, FILE_ERROR_MESSAGE_FIELD, ERROR_COUNT_FIELD, ERROR_FIELDS_FIELD, ERROR_TEXT_FIELD, + WARNING_FILES_TARGET_DIR, WARNING_FILES_EXTENTION, ERROR_FILES_TARGET_DIR, ERROR_FILES_EXTENTION, + LINE_NR_FILES_TARGET_DIR, LINE_NR_FILES_EXTENTION + ); + for ( OldTextFileInputMetaInjection.Entry entry : stringEntries ) { + generators.put( entry, stringGenerator ); + } + + Generator intGenerator = new ValidatorAdapter( new IntLoadSaveValidator() ); + List intEntries = asList( + NR_HEADER_LINES, NR_FOOTER_LINES, NR_WRAPS, NR_DOC_HEADER_LINES, NR_LINES_PER_PAGE, ROW_LIMIT + ); + for ( OldTextFileInputMetaInjection.Entry entry : intEntries ) { + generators.put( entry, intGenerator ); + } + + Generator yesNoGenerator = new YesNoGenerator(); + List yesNoEntries = asList( + BREAK_IN_ENCLOSURE, HEADER_PRESENT, HAS_FOOTER, HAS_WRAPPED_LINES, HAS_PAGED_LAYOUT, NO_EMPTY_LINES, + INCLUDE_FILENAME, INCLUDE_ROW_NUMBER, ROW_NUMBER_BY_FILE, DATE_FORMAT_LENIENT, ACCEPT_FILE_NAMES, + PASS_THROUGH_FIELDS, ADD_FILES_TO_RESULT, FILE_HIDDEN_FIELDNAME, SKIP_BAD_FILES, IGNORE_ERRORS, + ERROR_LINES_SKIPPED + ); + for ( OldTextFileInputMetaInjection.Entry entry : yesNoEntries ) { + generators.put( entry, yesNoGenerator ); + } + + generators.put( DATE_FORMAT_LOCALE, new Constant( "en" ) ); + + return generators; + } + + private static void assertInjected( List fields, List toBeInjected ) { + Map map = new HashMap( fields.size() ); + for ( StepInjectionMetaEntry field : fields ) { + map.put( field.getKey(), field ); + } + + for ( StepInjectionMetaEntry entry : toBeInjected ) { + StepInjectionMetaEntry field = map.get( entry.getKey() ); + assertNotNull( entry.getKey(), field ); + + Object value = field.getValue(); + if ( value == null ) { + assertNull( entry.getKey(), entry.getValue() ); + } else { + assertEquals( entry.getKey(), entry.getValue(), value ); + } + } + } + + + private static interface Generator { + T generateValue(); + } + + private static class YesNoGenerator implements Generator { + private final Random random = new Random(); + + @Override public String generateValue() { + if ( random.nextBoolean() ) { + return "Y"; + } else { + return "N"; + } + } + } + + private static class ValidatorAdapter implements Generator { + private final FieldLoadSaveValidator validator; + + public ValidatorAdapter( FieldLoadSaveValidator validator ) { + this.validator = validator; + } + + @Override public String generateValue() { + return validator.getTestObject().toString(); + } + } + + private static class Constant implements Generator { + private final T value; + + public Constant( T value ) { + this.value = value; + } + + @Override public T generateValue() { + return value; + } + } + +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaLoadSaveTest.java b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaLoadSaveTest.java new file mode 100644 index 000000000000..7aed04224679 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputMetaLoadSaveTest.java @@ -0,0 +1,210 @@ +/* ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import org.apache.commons.lang.builder.EqualsBuilder; +import org.junit.Before; +import org.junit.Test; +import org.pentaho.di.job.entry.loadSave.TransStepLoadSaveTester; +import org.pentaho.di.trans.steps.loadsave.validator.ArrayLoadSaveValidator; +import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import static org.junit.Assert.assertTrue; + +/** + * @author Andrey Khayrutdinov + */ +public class OldTextFileInputMetaLoadSaveTest { + + private TransStepLoadSaveTester tester; + + @Before + public void setUp() throws Exception { + List commonAttributes = Arrays.asList( + "fileName", + "fileType", + "separator", + "enclosure", + "escapeCharacter", + "breakInEnclosureAllowed", + "header", + "nrHeaderLines", + "footer", + "nrFooterLines", + "lineWrapped", + "nrWraps", + "layoutPaged", + "nrLinesDocHeader", + "nrLinesPerPage", + "fileCompression", + "noEmptyLines", + "includeFilename", + "filenameField", + "includeRowNumber", + "rowNumberByFile", + "rowNumberField", + "fileFormat", + "rowLimit", + "inputFields", + "filter", + "encoding", + "errorIgnored", + "errorCountField", + "errorFieldsField", + "errorTextField", + "warningFilesDestinationDirectory", + "warningFilesExtension", + "errorFilesDestinationDirectory", + "errorFilesExtension", + "lineNumberFilesDestinationDirectory", + "lineNumberFilesExtension", + "dateFormatLenient", + "dateFormatLocale", + "errorLineSkipped", + "acceptingFilenames", + "passingThruFields", + "acceptingField", + "isaddresult", + "shortFileFieldName", + "pathFieldName", + "hiddenFieldName", + "lastModificationTimeFieldName", + "uriNameFieldName", + "rootUriNameFieldName", + "extensionFieldName", + "sizeFieldName", + "skipBadFiles", + "fileErrorField", + "fileErrorMessageField" + ); + List xmlAttributes = Collections.emptyList(); + List repoAttributes = Collections.emptyList(); + + Map getters = new HashMap( ); + getters.put( "header", "hasHeader" ); + getters.put( "footer", "hasFooter" ); + getters.put( "noEmptyLines", "noEmptyLines" ); + getters.put( "includeFilename", "includeFilename" ); + getters.put( "includeRowNumber", "includeRowNumber" ); + getters.put( "errorFilesExtension", "getErrorLineFilesExtension" ); + getters.put( "isaddresult", "isAddResultFile" ); + getters.put( "shortFileFieldName", "getShortFileNameField" ); + getters.put( "pathFieldName", "getPathField" ); + getters.put( "hiddenFieldName", "isHiddenField" ); + getters.put( "lastModificationTimeFieldName", "getLastModificationDateField" ); + getters.put( "uriNameFieldName", "getUriField" ); + getters.put( "rootUriNameFieldName", "getRootUriField" ); + getters.put( "extensionFieldName", "getExtensionField" ); + getters.put( "sizeFieldName", "getSizeField" ); + + Map setters = new HashMap( ); + setters.put( "fileName", "setFileNameForTest" ); + setters.put( "errorFilesExtension", "setErrorLineFilesExtension" ); + setters.put( "isaddresult", "setAddResultFile" ); + setters.put( "shortFileFieldName", "setShortFileNameField" ); + setters.put( "pathFieldName", "setPathField" ); + setters.put( "hiddenFieldName", "setIsHiddenField" ); + setters.put( "lastModificationTimeFieldName", "setLastModificationDateField" ); + setters.put( "uriNameFieldName", "setUriField" ); + setters.put( "rootUriNameFieldName", "setRootUriField" ); + setters.put( "extensionFieldName", "setExtensionField" ); + setters.put( "sizeFieldName", "setSizeField" ); + + Map> attributeValidators = Collections.emptyMap(); + + Map> typeValidators = new HashMap>( ); + typeValidators.put( OldTextFileFilter[].class.getCanonicalName(), new ArrayLoadSaveValidator( new TextFileFilterValidator() ) ); + typeValidators.put( TextFileInputField[].class.getCanonicalName(), new ArrayLoadSaveValidator( new TextFileInputFieldValidator() ) ); + + assertTrue( !commonAttributes.isEmpty() || !( xmlAttributes.isEmpty() || repoAttributes.isEmpty() ) ); + + tester = + new TransStepLoadSaveTester( OldTextFileInputMeta.class, commonAttributes, xmlAttributes, + repoAttributes, getters, setters, attributeValidators, typeValidators ); + } + + @Test + public void xmlSerialization() throws Exception { + tester.testXmlRoundTrip(); + } + + @Test + public void repositorySerialization() throws Exception { + tester.testRepoRoundTrip(); + } + + + private static class TextFileInputFieldValidator implements FieldLoadSaveValidator { + @Override public TextFileInputField getTestObject() { + return new TextFileInputField( UUID.randomUUID().toString(), new Random().nextInt(), new Random().nextInt() ); + } + + @Override + public boolean validateTestObject( TextFileInputField testObject, Object actual ) { + if ( !( actual instanceof TextFileInputField ) ) { + return false; + } + + TextFileInputField another = (TextFileInputField) actual; + return new EqualsBuilder() + .append( testObject.getName(), another.getName() ) + .append( testObject.getLength(), another.getLength() ) + .append( testObject.getPosition(), another.getPosition() ) + .isEquals(); + } + } + + private static class TextFileFilterValidator implements FieldLoadSaveValidator { + @Override public OldTextFileFilter getTestObject() { + OldTextFileFilter fileFilter = new OldTextFileFilter(); + fileFilter.setFilterPosition( new Random().nextInt() ); + fileFilter.setFilterString( UUID.randomUUID().toString() ); + fileFilter.setFilterLastLine( new Random().nextBoolean() ); + fileFilter.setFilterPositive( new Random().nextBoolean() ); + return fileFilter; + } + + @Override public boolean validateTestObject( OldTextFileFilter testObject, Object actual ) { + if ( !( actual instanceof OldTextFileFilter ) ) { + return false; + } + + OldTextFileFilter another = (OldTextFileFilter) actual; + return new EqualsBuilder() + .append( testObject.getFilterPosition(), another.getFilterPosition() ) + .append( testObject.getFilterString(), another.getFilterString() ) + .append( testObject.isFilterLastLine(), another.isFilterLastLine() ) + .append( testObject.isFilterPositive(), another.isFilterPositive() ) + .isEquals(); + } + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputTest.java b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputTest.java new file mode 100644 index 000000000000..bfe4a1a9c35d --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/OldTextFileInputTest.java @@ -0,0 +1,189 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.util.Collections; + +import org.apache.commons.io.IOUtils; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.pentaho.di.core.BlockingRowSet; +import org.pentaho.di.core.KettleEnvironment; +import org.pentaho.di.core.RowSet; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.playlist.FilePlayListAll; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.value.ValueMetaString; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.steps.StepMockUtil; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; + +import static org.junit.Assert.*; + +public class OldTextFileInputTest { + + @BeforeClass + public static void initKettle() throws Exception { + KettleEnvironment.init(); + } + + private static InputStreamReader getInputStreamReader( String data ) throws UnsupportedEncodingException { + return new InputStreamReader( new ByteArrayInputStream( data.getBytes( ( "UTF-8" ) ) ) ); + } + + @Test + public void testGetLineDOS() throws KettleFileException, UnsupportedEncodingException { + String input = "col1\tcol2\tcol3\r\ndata1\tdata2\tdata3\r\n"; + String expected = "col1\tcol2\tcol3"; + String output = OldTextFileInput.getLine( null, getInputStreamReader( input ), + OldTextFileInputMeta.FILE_FORMAT_DOS, new StringBuilder( 1000 ) ); + assertEquals( expected, output ); + } + + @Test + public void testGetLineUnix() throws KettleFileException, UnsupportedEncodingException { + String input = "col1\tcol2\tcol3\ndata1\tdata2\tdata3\n"; + String expected = "col1\tcol2\tcol3"; + String output = OldTextFileInput.getLine( null, getInputStreamReader( input ), + OldTextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ); + assertEquals( expected, output ); + } + + @Test + public void testGetLineOSX() throws KettleFileException, UnsupportedEncodingException { + String input = "col1\tcol2\tcol3\rdata1\tdata2\tdata3\r"; + String expected = "col1\tcol2\tcol3"; + String output = OldTextFileInput.getLine( null, getInputStreamReader( input ), + OldTextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ); + assertEquals( expected, output ); + } + + @Test + public void testGetLineMixed() throws KettleFileException, UnsupportedEncodingException { + String input = "col1\tcol2\tcol3\r\ndata1\tdata2\tdata3\r"; + String expected = "col1\tcol2\tcol3"; + String output = OldTextFileInput.getLine( null, getInputStreamReader( input ), + OldTextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder( 1000 ) ); + assertEquals( expected, output ); + } + + @Test( timeout = 100 ) + public void test_PDI695() throws KettleFileException, UnsupportedEncodingException { + String inputDOS = "col1\tcol2\tcol3\r\ndata1\tdata2\tdata3\r\n"; + String inputUnix = "col1\tcol2\tcol3\ndata1\tdata2\tdata3\n"; + String inputOSX = "col1\tcol2\tcol3\rdata1\tdata2\tdata3\r"; + String expected = "col1\tcol2\tcol3"; + + assertEquals( expected, OldTextFileInput.getLine( null, getInputStreamReader( inputDOS ), + OldTextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + assertEquals( expected, OldTextFileInput.getLine( null, getInputStreamReader( inputUnix ), + OldTextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + assertEquals( expected, OldTextFileInput.getLine( null, getInputStreamReader( inputOSX ), + OldTextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + } + + @Test + public void readWrappedInputWithoutHeaders() throws Exception { + final String virtualFile = "ram://pdi-2607.txt"; + KettleVFS.getFileObject( virtualFile ).createFile(); + + final String content = new StringBuilder() + .append( "r1c1" ).append( '\n' ).append( ";r1c2\n" ) + .append( "r2c1" ).append( '\n' ).append( ";r2c2" ) + .toString(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + bos.write( content.getBytes() ); + + OutputStream os = KettleVFS.getFileObject( virtualFile ).getContent().getOutputStream(); + IOUtils.copy( new ByteArrayInputStream( bos.toByteArray() ), os ); + os.close(); + + + OldTextFileInputMeta meta = new OldTextFileInputMeta(); + meta.setLineWrapped( true ); + meta.setNrWraps( 1 ); + meta.setInputFields( new TextFileInputField[] { + new TextFileInputField( "col1", -1, -1 ), + new TextFileInputField( "col2", -1, -1 ) + } ); + meta.setFileCompression( "None" ); + meta.setFileType( "CSV" ); + meta.setHeader( false ); + meta.setNrHeaderLines( -1 ); + meta.setFooter( false ); + meta.setNrFooterLines( -1 ); + + OldTextFileInputData data = new OldTextFileInputData(); + data.setFiles( new FileInputList() ); + data.getFiles().addFile( KettleVFS.getFileObject( virtualFile ) ); + + data.outputRowMeta = new RowMeta(); + data.outputRowMeta.addValueMeta( new ValueMetaString( "col1" ) ); + data.outputRowMeta.addValueMeta( new ValueMetaString( "col2" ) ); + + data.dataErrorLineHandler = Mockito.mock( FileErrorHandler.class ); + data.fileFormatType = OldTextFileInputMeta.FILE_FORMAT_UNIX; + data.separator = ";"; + data.filterProcessor = new OldTextFileFilterProcessor( new OldTextFileFilter[ 0 ] ); + data.filePlayList = new FilePlayListAll(); + + + RowSet output = new BlockingRowSet( 5 ); + OldTextFileInput input = StepMockUtil.getStep( OldTextFileInput.class, OldTextFileInputMeta.class, "test" ); + input.setOutputRowSets( Collections.singletonList( output ) ); + while ( input.processRow( meta, data ) ) { + // wait until the step completes executing + } + + Object[] row1 = output.getRowImmediate(); + assertRow( row1, "r1c1", "r1c2" ); + + Object[] row2 = output.getRowImmediate(); + assertRow( row2, "r2c1", "r2c2" ); + + + KettleVFS.getFileObject( virtualFile ).delete(); + } + + private static void assertRow( Object[] row, Object... values ) { + assertNotNull( row ); + assertTrue( String.format( "%d < %d", row.length, values.length ), row.length >= values.length ); + int i = 0; + while ( i < values.length ) { + assertEquals( values[ i ], row[ i ] ); + i++; + } + while ( i < row.length ) { + assertNull( row[ i ] ); + i++; + } + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/PDI_2875_Test.java b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/PDI_2875_Test.java new file mode 100644 index 000000000000..40c6a9e42a6d --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/oldtextfileinput/PDI_2875_Test.java @@ -0,0 +1,81 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2014 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.trans.steps.oldtextfileinput; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.pentaho.di.core.KettleEnvironment; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.logging.LoggingObjectInterface; +import org.pentaho.di.trans.steps.mock.StepMockHelper; + +import java.util.Date; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +/** + * Test case for PDI-2875 + * + * @author Pavel Sakun + */ +public class PDI_2875_Test { + private static StepMockHelper smh; + private final String VAR_NAME = "VAR"; + private final String EXPRESSION = "${" + VAR_NAME + "}"; + + @BeforeClass + public static void setUp() throws KettleException { + KettleEnvironment.init(); + smh = + new StepMockHelper( "CsvInputTest", OldTextFileInputMeta.class, OldTextFileInputData.class ); + when( smh.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ) + .thenReturn( smh.logChannelInterface ); + when( smh.trans.isRunning() ).thenReturn( true ); + } + + private OldTextFileInputMeta getMeta() { + OldTextFileInputMeta meta = new OldTextFileInputMeta(); + meta.allocateFiles( 2 ); + meta.setFileName( new String[]{ "file1.txt", "file2.txt" } ); + meta.setIncludeSubFolders( new String[] {"n", "n"} ); + meta.setFilter( new OldTextFileFilter[0] ); + meta.setFileFormat( "unix" ); + meta.setFileType( "CSV" ); + meta.setLineNumberFilesDestinationDirectory( EXPRESSION ); + meta.setErrorFilesDestinationDirectory( EXPRESSION ); + meta.setWarningFilesDestinationDirectory( EXPRESSION ); + + return meta; + } + + @Test + public void testVariableSubstitution() { + doReturn( new Date() ).when( smh.trans ).getCurrentDate(); + OldTextFileInput step = spy( new OldTextFileInput( smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans ) ); + OldTextFileInputData data = new OldTextFileInputData(); + step.setVariable( VAR_NAME, "value" ); + step.init( getMeta(), data ); + verify( step, times( 2 ) ).environmentSubstitute( EXPRESSION ); + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/PDI_2875_Test.java b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/PDI_2875_Test.java index 8327eb320a4a..183f749788b8 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/PDI_2875_Test.java +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/PDI_2875_Test.java @@ -58,13 +58,13 @@ private TextFileInputMeta getMeta() { TextFileInputMeta meta = new TextFileInputMeta(); meta.allocateFiles( 2 ); meta.setFileName( new String[]{ "file1.txt", "file2.txt" } ); - meta.setIncludeSubFolders( new String[] {"n", "n"} ); + meta.inputFiles.includeSubFolders= new String[] {"n", "n"} ; meta.setFilter( new TextFileFilter[0] ); - meta.setFileFormat( "unix" ); - meta.setFileType( "CSV" ); - meta.setLineNumberFilesDestinationDirectory( EXPRESSION ); - meta.setErrorFilesDestinationDirectory( EXPRESSION ); - meta.setWarningFilesDestinationDirectory( EXPRESSION ); + meta.content.fileFormat = "unix" ; + meta.content.fileType = "CSV" ; + meta.errorHandling.lineNumberFilesDestinationDirectory= EXPRESSION ; + meta.errorHandling.errorFilesDestinationDirectory = EXPRESSION ; + meta.errorHandling.warningFilesDestinationDirectory= EXPRESSION ; return meta; } diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputContentParsingTest.java b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputContentParsingTest.java new file mode 100644 index 000000000000..a1b4d53949b8 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputContentParsingTest.java @@ -0,0 +1,184 @@ +package org.pentaho.di.trans.steps.textfileinput; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.commons.vfs2.VFS; +import org.junit.Before; +import org.junit.Test; +import org.pentaho.di.core.KettleEnvironment; +import org.pentaho.di.core.compress.CompressionPluginType; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.logging.LogChannel; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.playlist.FilePlayList; +import org.pentaho.di.core.plugins.PluginRegistry; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.variables.Variables; +import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; +import org.pentaho.di.trans.steps.baseinput.IBaseInputStepControl; + +public class TextFileInputContentParsingTest { + LogChannelInterface log = new LogChannel( "junit" ); + FileSystemManager fs; + String inPrefix; + TextFileInputMeta meta; + TextFileInputData data; + IBaseInputStepControl stepControl; + List rows = new ArrayList<>(); + int errorsCount; + + public void init() throws Exception { + TextFileInputMeta m = new TextFileInputMeta(); + m.setDefault(); + + init( m ); + } + + public void init( TextFileInputMeta meta ) throws Exception { + KettleEnvironment.init(); + PluginRegistry.addPluginType( CompressionPluginType.getInstance() ); + PluginRegistry.init( true ); + + inPrefix = '/' + TextFileInputContentParsingTest.class.getPackage().getName().replace( '.', '/' ) + "/texts/"; + + fs = VFS.getManager(); + + meta = new TextFileInputMeta(); + + meta.setDefault(); + + data = new TextFileInputData(); + + data.outputRowMeta = new RowMeta(); + + data.separator = meta.content.separator; + data.enclosure = meta.content.enclosure; + data.escapeCharacter = meta.content.escapeCharacter; + + data.filterProcessor = new TextFileFilterProcessor( new TextFileFilter[0], new Variables() ); + data.dataErrorLineHandler = new FileErrorHandler() { + @Override + public void handleNonExistantFile( FileObject file ) throws KettleException { + errorsCount++; + } + + @Override + public void handleNonAccessibleFile( FileObject file ) throws KettleException { + errorsCount++; + } + + @Override + public void handleLineError( long lineNr, String filePart ) throws KettleException { + errorsCount++; + } + + @Override + public void handleFile( FileObject file ) throws KettleException { + } + + @Override + public void close() throws KettleException { + } + }; + + stepControl = new IBaseInputStepControl() { + + @Override + public void stopAll() { + } + + @Override + public void setErrors( long e ) { + } + + @Override + public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { + rows.add( Arrays.copyOf( row, rowMeta.size() ) ); + } + + @Override + public long incrementLinesUpdated() { + return 0; + } + + @Override + public long incrementLinesInput() { + return 0; + } + + @Override + public long getLinesWritten() { + return 0; + } + + @Override + public long getLinesInput() { + return 0; + } + + @Override + public long getErrors() { + return 0; + } + + @Override + public boolean failAfterBadFile( String errorMsg ) { + return false; + } + + @Override + public boolean checkFeedback( long lines ) { + return false; + } + }; + data.filePlayList = new FilePlayList() { + public boolean isProcessingNeeded( FileObject file, long lineNr, String filePart ) throws KettleException { + return true; + } + }; + } + + void check( Object[][] expected ) { + assertEquals( "There are errors", 0, errorsCount ); + assertEquals( "Wrong rows count", expected.length, rows.size() ); + for ( int i = 0; i < expected.length; i++ ) { + assertArrayEquals( "Wrong row: " + Arrays.asList( rows.get( i ) ), expected[i], rows.get( i ) ); + } + } + + FileObject getFile( String filename ) throws Exception { + FileObject file = fs.resolveFile( this.getClass().getResource( inPrefix + filename ).toExternalForm() ); + assertNotNull( "There is no file", file ); + return file; + } + + void setFields( TextFileInputField... fields ) throws Exception { + meta.inputFiles.inputFields = fields; + meta.getFields( data.outputRowMeta, meta.getName(), null, null, new Variables(), null, null ); + data.convertRowMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); + } + + @Test + public void defaultOptions() throws Exception { + setFields( new TextFileInputField(), new TextFileInputField(), new TextFileInputField() ); + + try (TextFileInputReader reader = new TextFileInputReader( stepControl, meta, data, getFile( "default.csv" ), log )) { + while ( reader.readRow() ) + ; + } + + // compare rows + check( new Object[][] { { "first", "1", "1.1" }, { "second", "2", "2.2" }, { "third", "3", "3.3" } } ); + } +} diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMetaInjectionTest.java b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMetaInjectionTest.java index 66d4f12f7f58..89d837d3f232 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMetaInjectionTest.java +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputMetaInjectionTest.java @@ -22,12 +22,11 @@ package org.pentaho.di.trans.steps.textfileinput; -import org.junit.Test; -import org.pentaho.di.trans.step.StepInjectionMetaEntry; -import org.pentaho.di.trans.step.StepInjectionUtil; -import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; -import org.pentaho.di.trans.steps.loadsave.validator.IntLoadSaveValidator; -import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.pentaho.di.core.row.ValueMetaInterface.TYPE_NONE; import java.util.ArrayList; import java.util.EnumMap; @@ -36,9 +35,12 @@ import java.util.Map; import java.util.Random; -import static java.util.Arrays.asList; -import static org.junit.Assert.*; -import static org.pentaho.di.core.row.ValueMetaInterface.TYPE_NONE; +import org.junit.Test; +import org.pentaho.di.trans.step.StepInjectionMetaEntry; +import org.pentaho.di.trans.step.StepInjectionUtil; +import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; +import org.pentaho.di.trans.steps.loadsave.validator.IntLoadSaveValidator; +import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator; import static org.pentaho.di.trans.steps.textfileinput.TextFileInputMetaInjection.Entry.*; public class TextFileInputMetaInjectionTest { diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputTest.java b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputTest.java index b0ca4344675f..871ee2ec15b2 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputTest.java +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/TextFileInputTest.java @@ -41,6 +41,7 @@ import org.pentaho.di.core.playlist.FilePlayListAll; import org.pentaho.di.core.row.RowMeta; import org.pentaho.di.core.row.value.ValueMetaString; +import org.pentaho.di.core.variables.Variables; import org.pentaho.di.core.vfs.KettleVFS; import org.pentaho.di.trans.step.errorhandling.FileErrorHandler; import org.pentaho.di.trans.steps.StepMockUtil; @@ -62,8 +63,9 @@ private static InputStreamReader getInputStreamReader( String data ) throws Unsu public void testGetLineDOS() throws KettleFileException, UnsupportedEncodingException { String input = "col1\tcol2\tcol3\r\ndata1\tdata2\tdata3\r\n"; String expected = "col1\tcol2\tcol3"; - String output = TextFileInput.getLine( null, getInputStreamReader( input ), - TextFileInputMeta.FILE_FORMAT_DOS, new StringBuilder( 1000 ) ); + String output = + TextFileInputUtils.getLine( null, getInputStreamReader( input ), TextFileInputMeta.FILE_FORMAT_DOS, + new StringBuilder( 1000 ) ); assertEquals( expected, output ); } @@ -71,8 +73,9 @@ public void testGetLineDOS() throws KettleFileException, UnsupportedEncodingExce public void testGetLineUnix() throws KettleFileException, UnsupportedEncodingException { String input = "col1\tcol2\tcol3\ndata1\tdata2\tdata3\n"; String expected = "col1\tcol2\tcol3"; - String output = TextFileInput.getLine( null, getInputStreamReader( input ), - TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ); + String output = + TextFileInputUtils.getLine( null, getInputStreamReader( input ), TextFileInputMeta.FILE_FORMAT_UNIX, + new StringBuilder( 1000 ) ); assertEquals( expected, output ); } @@ -80,8 +83,9 @@ public void testGetLineUnix() throws KettleFileException, UnsupportedEncodingExc public void testGetLineOSX() throws KettleFileException, UnsupportedEncodingException { String input = "col1\tcol2\tcol3\rdata1\tdata2\tdata3\r"; String expected = "col1\tcol2\tcol3"; - String output = TextFileInput.getLine( null, getInputStreamReader( input ), - TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ); + String output = + TextFileInputUtils.getLine( null, getInputStreamReader( input ), TextFileInputMeta.FILE_FORMAT_UNIX, + new StringBuilder( 1000 ) ); assertEquals( expected, output ); } @@ -89,8 +93,9 @@ public void testGetLineOSX() throws KettleFileException, UnsupportedEncodingExce public void testGetLineMixed() throws KettleFileException, UnsupportedEncodingException { String input = "col1\tcol2\tcol3\r\ndata1\tdata2\tdata3\r"; String expected = "col1\tcol2\tcol3"; - String output = TextFileInput.getLine( null, getInputStreamReader( input ), - TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder( 1000 ) ); + String output = + TextFileInputUtils.getLine( null, getInputStreamReader( input ), TextFileInputMeta.FILE_FORMAT_MIXED, + new StringBuilder( 1000 ) ); assertEquals( expected, output ); } @@ -101,12 +106,12 @@ public void test_PDI695() throws KettleFileException, UnsupportedEncodingExcepti String inputOSX = "col1\tcol2\tcol3\rdata1\tdata2\tdata3\r"; String expected = "col1\tcol2\tcol3"; - assertEquals( expected, TextFileInput.getLine( null, getInputStreamReader( inputDOS ), - TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); - assertEquals( expected, TextFileInput.getLine( null, getInputStreamReader( inputUnix ), - TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); - assertEquals( expected, TextFileInput.getLine( null, getInputStreamReader( inputOSX ), - TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + assertEquals( expected, TextFileInputUtils.getLine( null, getInputStreamReader( inputDOS ), + TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + assertEquals( expected, TextFileInputUtils.getLine( null, getInputStreamReader( inputUnix ), + TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); + assertEquals( expected, TextFileInputUtils.getLine( null, getInputStreamReader( inputOSX ), + TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ) ); } @Test @@ -114,10 +119,9 @@ public void readWrappedInputWithoutHeaders() throws Exception { final String virtualFile = "ram://pdi-2607.txt"; KettleVFS.getFileObject( virtualFile ).createFile(); - final String content = new StringBuilder() - .append( "r1c1" ).append( '\n' ).append( ";r1c2\n" ) - .append( "r2c1" ).append( '\n' ).append( ";r2c2" ) - .toString(); + final String content = + new StringBuilder().append( "r1c1" ).append( '\n' ).append( ";r1c2\n" ).append( "r2c1" ).append( '\n' ).append( + ";r2c2" ).toString(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write( content.getBytes() ); @@ -125,24 +129,21 @@ public void readWrappedInputWithoutHeaders() throws Exception { IOUtils.copy( new ByteArrayInputStream( bos.toByteArray() ), os ); os.close(); - TextFileInputMeta meta = new TextFileInputMeta(); - meta.setLineWrapped( true ); - meta.setNrWraps( 1 ); - meta.setInputFields( new TextFileInputField[] { - new TextFileInputField( "col1", -1, -1 ), - new TextFileInputField( "col2", -1, -1 ) - } ); - meta.setFileCompression( "None" ); - meta.setFileType( "CSV" ); - meta.setHeader( false ); - meta.setNrHeaderLines( -1 ); - meta.setFooter( false ); - meta.setNrFooterLines( -1 ); + meta.content.lineWrapped = true ; + meta.content.nrWraps = 1; + meta.inputFiles.inputFields = + new TextFileInputField[] { new TextFileInputField( "col1", -1, -1 ), new TextFileInputField( "col2", -1, -1 ) }; + meta.content.fileCompression = "None" ; + meta.content.fileType = "CSV" ; + meta.content.header= false ; + meta.content.nrHeaderLines= -1 ; + meta.content.footer =false ; + meta.content.nrFooterLines = -1 ; TextFileInputData data = new TextFileInputData(); - data.setFiles( new FileInputList() ); - data.getFiles().addFile( KettleVFS.getFileObject( virtualFile ) ); + data.files = new FileInputList(); + data.files.addFile( KettleVFS.getFileObject( virtualFile ) ); data.outputRowMeta = new RowMeta(); data.outputRowMeta.addValueMeta( new ValueMetaString( "col1" ) ); @@ -151,10 +152,9 @@ public void readWrappedInputWithoutHeaders() throws Exception { data.dataErrorLineHandler = Mockito.mock( FileErrorHandler.class ); data.fileFormatType = TextFileInputMeta.FILE_FORMAT_UNIX; data.separator = ";"; - data.filterProcessor = new TextFileFilterProcessor( new TextFileFilter[ 0 ] ); + data.filterProcessor = new TextFileFilterProcessor( new TextFileFilter[0], new Variables() ); data.filePlayList = new FilePlayListAll(); - RowSet output = new BlockingRowSet( 5 ); TextFileInput input = StepMockUtil.getStep( TextFileInput.class, TextFileInputMeta.class, "test" ); input.setOutputRowSets( Collections.singletonList( output ) ); @@ -168,7 +168,6 @@ public void readWrappedInputWithoutHeaders() throws Exception { Object[] row2 = output.getRowImmediate(); assertRow( row2, "r2c1", "r2c2" ); - KettleVFS.getFileObject( virtualFile ).delete(); } @@ -177,11 +176,11 @@ private static void assertRow( Object[] row, Object... values ) { assertTrue( String.format( "%d < %d", row.length, values.length ), row.length >= values.length ); int i = 0; while ( i < values.length ) { - assertEquals( values[ i ], row[ i ] ); + assertEquals( values[i], row[i] ); i++; } while ( i < row.length ) { - assertNull( row[ i ] ); + assertNull( row[i] ); i++; } } diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/.gitattributes b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/.gitattributes new file mode 100644 index 000000000000..206ae01d7141 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/.gitattributes @@ -0,0 +1 @@ +default.csv text eol=crlf diff --git a/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/default.csv b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/default.csv new file mode 100644 index 000000000000..05903c8245b9 --- /dev/null +++ b/engine/test-src/org/pentaho/di/trans/steps/textfileinput/texts/default.csv @@ -0,0 +1,4 @@ +Field 1;Field 2;Field 3 +first;1;1.1 +second;2;2.2 +third;3;3.3 diff --git a/plugins/kettle-s3csvinput-plugin/src/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputDialog.java b/plugins/kettle-s3csvinput-plugin/src/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputDialog.java index 42b7d3bced85..e965b88f075e 100644 --- a/plugins/kettle-s3csvinput-plugin/src/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputDialog.java +++ b/plugins/kettle-s3csvinput-plugin/src/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputDialog.java @@ -69,6 +69,7 @@ import org.pentaho.di.trans.steps.textfileinput.TextFileInput; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputUtils; import org.pentaho.di.ui.core.dialog.EnterNumberDialog; import org.pentaho.di.ui.core.dialog.EnterSelectionDialog; import org.pentaho.di.ui.core.dialog.EnterTextDialog; @@ -80,6 +81,7 @@ import org.pentaho.di.ui.core.widget.TextVar; import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog; import org.pentaho.di.ui.trans.step.BaseStepDialog; +import org.pentaho.di.ui.trans.steps.oldtextfileinput.OldTextFileCSVImportProgressDialog; import org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog; public class S3CsvInputDialog extends BaseStepDialog implements StepDialogInterface @@ -812,7 +814,7 @@ private void getCSV() // Read a line of data to determine the number of rows... // - String line = TextFileInput.getLine(log, reader, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder(1000)); + String line = TextFileInputUtils.getLine(log, reader, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder(1000)); // Split the string, header or data into parts... // @@ -855,7 +857,7 @@ private void getCSV() getInfo(meta); - TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true); + OldTextFileCSVImportProgressDialog pd = new OldTextFileCSVImportProgressDialog(shell, meta, transMeta, reader, samples, true); String message = pd.open(); if (message!=null) { diff --git a/ui/src/org/pentaho/di/ui/core/widget/OldTableDraw.java b/ui/src/org/pentaho/di/ui/core/widget/OldTableDraw.java new file mode 100644 index 000000000000..1334e6d2aea7 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/core/widget/OldTableDraw.java @@ -0,0 +1,541 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.core.widget; + +import java.util.List; +import java.util.Vector; + +import org.eclipse.jface.wizard.WizardPage; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.DisposeEvent; +import org.eclipse.swt.events.DisposeListener; +import org.eclipse.swt.events.MouseAdapter; +import org.eclipse.swt.events.MouseEvent; +import org.eclipse.swt.events.MouseMoveListener; +import org.eclipse.swt.events.PaintEvent; +import org.eclipse.swt.events.PaintListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.graphics.Color; +import org.eclipse.swt.graphics.GC; +import org.eclipse.swt.graphics.Image; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.graphics.Rectangle; +import org.eclipse.swt.widgets.Canvas; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.ScrollBar; +import org.pentaho.di.core.gui.TextFileInputFieldInterface; +import org.pentaho.di.ui.core.PropsUI; +import org.pentaho.di.ui.core.gui.GUIResource; + +/** + * Widget to draw the character of a fixed length text-file in a graphical way. + * + * @author Matt + * @since 17-04-2004 + */ +public class OldTableDraw extends Canvas { + private Display display; + private Color bg; + // private Font font; + private Color black, red, blue, lgray; + private Point offset; + private ScrollBar hori; + private ScrollBar vert; + private Image dummy_image; + private GC dummy_gc; + private int maxlen; + + private int fontheight; + private int fontwidth; + + private Image cache_image; + private int prev_fromx, prev_tox, prev_fromy, prev_toy; + + private Vector fields; + + private List rows; + + private static final int LEFT = 50; + private static final int TOP = 30; + private static final int MARGIN = 10; + + private int potential_click; + + private WizardPage wPage; + private String prevfieldname; + + public OldTableDraw( Composite parent, PropsUI props, WizardPage wPage, Vector fields ) { + super( parent, SWT.NO_BACKGROUND | SWT.H_SCROLL | SWT.V_SCROLL ); + this.wPage = wPage; + this.fields = fields; + + prevfieldname = ""; + + potential_click = -1; + + // Cache displayed text... + cache_image = null; + prev_fromx = -1; + prev_tox = -1; + prev_fromy = -1; + prev_toy = -1; + + display = parent.getDisplay(); + bg = GUIResource.getInstance().getColorBackground(); + // font = GUIResource.getInstance().getFontGrid(); + fontheight = props.getGridFont().getHeight(); + + black = GUIResource.getInstance().getColorBlack(); + red = GUIResource.getInstance().getColorRed(); + blue = GUIResource.getInstance().getColorBlue(); + lgray = GUIResource.getInstance().getColorLightGray(); + + hori = getHorizontalBar(); + vert = getVerticalBar(); + + // Determine font width... + dummy_image = new Image( display, 1, 1 ); + dummy_gc = new GC( dummy_image ); + // dummy_gc.setFont(font); + String teststring = "ABCDEF"; + fontwidth = Math.round( dummy_gc.textExtent( teststring ).x / teststring.length() ); + + setBackground( bg ); + // setFont(font); + + addPaintListener( new PaintListener() { + public void paintControl( PaintEvent e ) { + OldTableDraw.this.paintControl( e ); + } + } ); + + addDisposeListener( new DisposeListener() { + public void widgetDisposed( DisposeEvent arg0 ) { + dummy_gc.dispose(); + dummy_image.dispose(); + + if ( cache_image != null ) { + cache_image.dispose(); + } + } + } ); + + hori.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + redraw(); + } + } ); + vert.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + redraw(); + } + } ); + hori.setThumb( 100 ); + vert.setThumb( 100 ); + + // Mouse events! + + addMouseListener( new MouseAdapter() { + public void mouseDown( MouseEvent e ) { + Point offset = getOffset(); + int posx = (int) Math.round( (double) ( e.x - LEFT - MARGIN - offset.x ) / ( (double) fontwidth ) ); + + if ( posx > 0 ) { + potential_click = posx; + + redraw(); + } + } + + public void mouseUp( MouseEvent e ) { + if ( potential_click > 0 ) { + setMarker( potential_click ); // clear or set! + potential_click = -1; + + redraw(); + } + } + } ); + + addMouseMoveListener( new MouseMoveListener() { + public void mouseMove( MouseEvent e ) { + int posx = (int) Math.round( (double) ( e.x - LEFT - MARGIN - offset.x ) / ( (double) fontwidth ) ); + + // Clicked and mouse is down: move marker to a new location... + if ( potential_click >= 0 ) { + if ( posx > 0 ) { + potential_click = posx; + redraw(); + } + } + + TextFileInputFieldInterface field = getFieldOnPosition( posx ); + if ( field != null && !field.getName().equalsIgnoreCase( prevfieldname ) ) { + setToolTipText( field.getName() + " : length=" + field.getLength() ); + prevfieldname = field.getName(); + } + } + } ); + } + + private TextFileInputFieldInterface getFieldOnPosition( int x ) { + for ( int i = 0; i < fields.size(); i++ ) { + TextFileInputFieldInterface field = fields.get( i ); + int pos = field.getPosition(); + int len = field.getLength(); + if ( pos <= x && pos + len > x ) { + return field; + } + } + return null; + + } + + private void setMarker( int x ) { + int idx = -1; + int highest_smaller = -1; + + for ( int i = 0; i < fields.size(); i++ ) { + TextFileInputFieldInterface field = fields.get( i ); + + int pos = field.getPosition(); + int len = field.getLength(); + + if ( pos == potential_click ) { + idx = i; + } + if ( highest_smaller < 0 && pos + len >= x ) { + highest_smaller = i; // The first time this occurs is OK. + } + } + + // OK, so we need to add a new field on the location of the previous one + // Actually, we make the previous field shorter + // + // Field 1: pos=0, length=100 + // + // becomes + // + // Field1: pos=0, length=50 + // Field2: pos=51, length=50 + // + // We know the number of the new field : lowest_larger. + // + // Note: We should always have one field @ position 0, length max + // + if ( idx < 0 ) // Position is not yet in the list, the field is not deleted, but added + { + if ( highest_smaller >= 0 ) { + // OK, let's add a new field, but split the length of the previous field + // We want to keep this list sorted, so add at position lowest_larger. + // We want to change the previous entry and add another after it. + TextFileInputFieldInterface prevfield = fields.get( highest_smaller ); + int newlength = prevfield.getLength() - ( x - prevfield.getPosition() ); + TextFileInputFieldInterface field = prevfield.createNewInstance( getNewFieldname(), x, newlength ); + fields.add( highest_smaller + 1, field ); + + // Don't forget to make the previous field shorter + prevfield.setLength( x - prevfield.getPosition() ); + } + } else { + if ( highest_smaller >= 0 ) { + // Now we need to remove the field with the same starting position + // The previous field need to receive extra length + TextFileInputFieldInterface prevfield = fields.get( highest_smaller ); + TextFileInputFieldInterface field = fields.get( idx ); + prevfield.setLength( prevfield.getLength() + field.getLength() ); + // Remove the field + fields.remove( idx ); + } + } + + // Something has changed: change wizard page... + wPage.setPageComplete( wPage.canFlipToNextPage() ); + } + + // Loop from 1 to ... and see if we have an empty Fieldnr available. + private String getNewFieldname() { + int nr = 1; + String name = "Field" + nr; + while ( fieldExists( name ) ) { + nr++; + name = "Field" + nr; + } + return name; + } + + private boolean fieldExists( String name ) { + for ( int i = 0; i < fields.size(); i++ ) { + TextFileInputFieldInterface field = fields.get( i ); + + if ( name.equalsIgnoreCase( field.getName() ) ) { + return true; + } + } + return false; + } + + public void setRows( List rows ) { + this.rows = rows; + maxlen = getMaxLength(); + redraw(); + } + + // Draw the widget. + public void paintControl( PaintEvent e ) { + offset = getOffset(); + if ( offset == null ) { + return; + } + + Point area = getArea(); + Point max = getMaximum(); + Point thumb = getThumb( area, max ); + + hori.setThumb( thumb.x ); + vert.setThumb( thumb.y ); + + // From where do we need to draw? + int fromy = -offset.y / ( fontheight + 2 ); + int toy = fromy + ( area.y / ( fontheight + 2 ) ); + int fromx = -offset.x / fontwidth; + int tox = fromx + ( area.x / fontwidth ); + + Image image = new Image( display, area.x, area.y ); + + if ( fromx != prev_fromx || fromy != prev_fromy || tox != prev_tox || toy != prev_toy ) { + if ( cache_image != null ) { + cache_image.dispose(); + cache_image = null; + } + cache_image = new Image( display, area.x, area.y ); + + GC gc = new GC( cache_image ); + + // We have a cached image: draw onto it! + int linepos = TOP - 5; + + gc.setBackground( bg ); + gc.fillRectangle( LEFT, TOP, area.x, area.y ); + + // We draw in black... + gc.setForeground( black ); + // gc.setFont(font); + + // Draw the text + // + for ( int i = fromy; i < rows.size() && i < toy; i++ ) { + String str = rows.get( i ); + for ( int p = fromx; p < str.length() && p < tox; p++ ) { + gc.drawText( "" + str.charAt( p ), LEFT + MARGIN + p * fontwidth + offset.x, TOP + + i * ( fontheight + 2 ) + offset.y, true ); + } + + if ( str.length() < tox ) { + gc.setForeground( red ); + gc.setBackground( red ); + int x_oval = LEFT + MARGIN + str.length() * fontwidth + offset.x; + int y_oval = TOP + i * ( fontheight + 2 ) + offset.y; + gc.drawOval( x_oval, y_oval, fontwidth, fontheight ); + gc.fillOval( x_oval, y_oval, fontwidth, fontheight ); + gc.setForeground( black ); + gc.setBackground( bg ); + } + } + + // Draw the rulers + // HORIZONTAL + gc.setBackground( lgray ); + gc.fillRectangle( LEFT + MARGIN, 0, area.x, linepos + 1 ); + gc.setBackground( bg ); + + gc.drawLine( LEFT + MARGIN, linepos, area.x, linepos ); + + // Little tabs, small ones, every 5 big one, every 10 the number above... + for ( int i = fromx; i < maxlen + 10 && i < tox + 10; i++ ) { + String number = "" + i; + int numsize = number.length() * fontwidth; + + if ( i > 0 && ( i % 10 ) == 0 ) { + gc.drawText( + "" + i, LEFT + MARGIN + i * fontwidth - numsize / 2 + offset.x, linepos - 10 - fontheight, true ); + } + + if ( i > 0 && ( i % 5 ) == 0 ) { + gc.drawLine( + LEFT + MARGIN + i * fontwidth + offset.x, linepos, LEFT + MARGIN + i * fontwidth + offset.x, + linepos - 5 ); + } else { + gc.drawLine( + LEFT + MARGIN + i * fontwidth + offset.x, linepos, LEFT + MARGIN + i * fontwidth + offset.x, + linepos - 3 ); + } + } + + // VERTICIAL + gc.setBackground( lgray ); + gc.fillRectangle( 0, TOP, LEFT, area.y ); + + gc.drawLine( LEFT, TOP, LEFT, area.y ); + + for ( int i = fromy; i < rows.size() && i < toy; i++ ) { + String number = "" + ( i + 1 ); + int numsize = number.length() * fontwidth; + gc.drawText( number, LEFT - 5 - numsize, TOP + i * ( fontheight + 2 ) + offset.y, true ); + gc.drawLine( LEFT, TOP + ( i + 1 ) * ( fontheight + 2 ) + offset.y, LEFT - 5, TOP + + ( i + 1 ) * ( fontheight + 2 ) + offset.y ); + } + + gc.dispose(); + } + + GC gc = new GC( image ); + + // Draw the cached image onto the canvas image: + gc.drawImage( cache_image, 0, 0 ); + + // Also draw the markers... + gc.setForeground( red ); + gc.setBackground( red ); + for ( int i = 0; i < fields.size(); i++ ) { + int x = ( fields.get( i ) ).getPosition(); + if ( x >= fromx && x <= tox ) { + drawMarker( gc, x, area.y ); + } + } + if ( potential_click >= 0 ) { + gc.setForeground( blue ); + gc.setBackground( blue ); + drawMarker( gc, potential_click, area.y ); + } + + // Draw the image: + e.gc.drawImage( image, 0, 0 ); + gc.dispose(); + image.dispose(); + } + + private void drawMarker( GC gc, int x, int maxy ) { + int[] triangle = + new int[] { + LEFT + MARGIN + x * fontwidth + offset.x, TOP - 4, LEFT + MARGIN + x * fontwidth + offset.x + 3, + TOP + 1, LEFT + MARGIN + x * fontwidth + offset.x - 3, TOP + 1 }; + gc.fillPolygon( triangle ); + gc.drawPolygon( triangle ); + gc + .drawLine( + LEFT + MARGIN + x * fontwidth + offset.x, TOP + 1, LEFT + MARGIN + x * fontwidth + offset.x, maxy ); + } + + private Point getOffset() { + Point area = getArea(); + Point max = getMaximum(); + Point thumb = getThumb( area, max ); + Point offset = getOffset( thumb, area ); + + return offset; + } + + private Point getThumb( Point area, Point max ) { + Point thumb = new Point( 0, 0 ); + if ( max.x <= area.x ) { + thumb.x = 100; + } else { + thumb.x = Math.round( 100 * area.x / max.x ); + } + if ( max.y <= area.y ) { + thumb.y = 100; + } else { + thumb.y = Math.round( 100 * area.y / max.y ); + } + + return thumb; + } + + private Point getOffset( Point thumb, Point area ) { + Point p = new Point( 0, 0 ); + Point sel = new Point( hori.getSelection(), vert.getSelection() ); + + if ( thumb.x == 0 || thumb.y == 0 ) { + return p; + } + + p.x = Math.round( -sel.x * area.x / thumb.x ); + p.y = Math.round( -sel.y * area.y / thumb.y ); + + return p; + } + + private Point getMaximum() { + int maxx = 0; + int maxy = ( rows.size() + 10 ) * ( fontheight + 2 ); + + for ( int i = 0; i < rows.size(); i++ ) { + String str = rows.get( i ); + int len = ( str.length() + 10 ) * fontwidth; + + if ( maxx < len ) { + maxx = len; + } + } + + return new Point( maxx, maxy ); + } + + private int getMaxLength() { + int maxx = 0; + + for ( int i = 0; i < rows.size(); i++ ) { + String str = rows.get( i ); + int len = str.length(); + + if ( maxx < len ) { + maxx = len; + } + } + + return maxx; + } + + private Point getArea() { + Rectangle rect = getClientArea(); + Point area = new Point( rect.width, rect.height ); + + return area; + } + + public Vector getFields() { + return fields; + } + + public void setFields( Vector fields ) { + this.fields = fields; + } + + public void clearFields() { + fields = new Vector(); + } + +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/csvinput/CsvInputDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/csvinput/CsvInputDialog.java index cf51aa7cac2e..071ed9d729a1 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/csvinput/CsvInputDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/csvinput/CsvInputDialog.java @@ -78,9 +78,9 @@ import org.pentaho.di.trans.steps.csvinput.CsvInput; import org.pentaho.di.trans.steps.csvinput.CsvInputMeta; import org.pentaho.di.trans.steps.textfileinput.EncodingType; -import org.pentaho.di.trans.steps.textfileinput.TextFileInput; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputUtils; import org.pentaho.di.ui.core.PropsUI; import org.pentaho.di.ui.core.dialog.EnterNumberDialog; import org.pentaho.di.ui.core.dialog.EnterTextDialog; @@ -95,7 +95,7 @@ import org.pentaho.di.ui.spoon.trans.TransGraph; import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog; import org.pentaho.di.ui.trans.step.BaseStepDialog; -import org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog; +import org.pentaho.di.ui.trans.steps.oldtextfileinput.OldTextFileCSVImportProgressDialog; public class CsvInputDialog extends BaseStepDialog implements StepDialogInterface { private static Class PKG = CsvInput.class; // for i18n purposes, needed by Translator2!! @@ -884,8 +884,8 @@ private void getCSV() { // Read a line of data to determine the number of rows... // String line = - TextFileInput.getLine( - log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( 1000 ) ); + TextFileInputUtils.getLine( log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder( + 1000 ) ); // Split the string, header or data into parts... // @@ -938,8 +938,8 @@ private void getCSV() { if ( samples >= 0 ) { getInfo( meta ); - TextFileCSVImportProgressDialog pd = - new TextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, true ); + OldTextFileCSVImportProgressDialog pd = + new OldTextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, true ); String message = pd.open(); if ( message != null ) { wFields.removeAll(); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldDirectoryDialogButtonListenerFactory.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldDirectoryDialogButtonListenerFactory.java new file mode 100644 index 000000000000..ad10f30d377f --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldDirectoryDialogButtonListenerFactory.java @@ -0,0 +1,51 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.widgets.DirectoryDialog; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Text; + +public class OldDirectoryDialogButtonListenerFactory { + public static final SelectionAdapter getSelectionAdapter( final Shell shell, final Text destination ) { + // Listen to the Browse... button + return new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + DirectoryDialog dialog = new DirectoryDialog( shell, SWT.OPEN ); + if ( destination.getText() != null ) { + String fpath = destination.getText(); + // String fpath = StringUtil.environmentSubstitute(destination.getText()); + dialog.setFilterPath( fpath ); + } + + if ( dialog.open() != null ) { + String str = dialog.getFilterPath(); + destination.setText( str ); + } + } + }; + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileCSVImportProgressDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileCSVImportProgressDialog.java new file mode 100644 index 000000000000..942cf757bf71 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileCSVImportProgressDialog.java @@ -0,0 +1,498 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import java.io.InputStreamReader; +import java.lang.reflect.InvocationTargetException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.jface.dialogs.ProgressMonitorDialog; +import org.eclipse.jface.operation.IRunnableWithProgress; +import org.eclipse.swt.widgets.Shell; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.logging.LogChannel; +import org.pentaho.di.core.logging.LogChannelInterface; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.util.StringEvaluationResult; +import org.pentaho.di.core.util.StringEvaluator; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.steps.oldtextfileinput.OldEncodingType; +import org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileInput; +import org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileLine; +import org.pentaho.di.trans.steps.textfileinput.InputFileMetaInterface; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.ui.core.dialog.ErrorDialog; + +/** + * Takes care of displaying a dialog that will handle the wait while we're finding out what tables, views etc we can + * reach in the database. + * + * @author Matt + * @since 07-apr-2005 + */ +public class OldTextFileCSVImportProgressDialog { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + private Shell shell; + + private InputFileMetaInterface meta; + + private int samples; + + private boolean replaceMeta; + + private String message; + + private String debug; + + private long rownumber; + + private InputStreamReader reader; + + private TransMeta transMeta; + + private LogChannelInterface log; + + private OldEncodingType encodingType; + + /** + * Creates a new dialog that will handle the wait while we're finding out what tables, views etc we can reach in the + * database. + */ + public OldTextFileCSVImportProgressDialog( Shell shell, InputFileMetaInterface meta, TransMeta transMeta, + InputStreamReader reader, int samples, boolean replaceMeta ) { + this.shell = shell; + this.meta = meta; + this.reader = reader; + this.samples = samples; + this.replaceMeta = replaceMeta; + this.transMeta = transMeta; + + message = null; + debug = "init"; + rownumber = 1L; + + this.log = new LogChannel( transMeta ); + + this.encodingType = OldEncodingType.guessEncodingType( reader.getEncoding() ); + + } + + public String open() { + IRunnableWithProgress op = new IRunnableWithProgress() { + public void run( IProgressMonitor monitor ) throws InvocationTargetException, InterruptedException { + try { + message = doScan( monitor ); + } catch ( Exception e ) { + e.printStackTrace(); + throw new InvocationTargetException( e, + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Exception.ErrorScanningFile", + "" + rownumber, debug, e.toString() ) ); + } + } + }; + + try { + ProgressMonitorDialog pmd = new ProgressMonitorDialog( shell ); + pmd.run( true, true, op ); + } catch ( InvocationTargetException e ) { + new ErrorDialog( shell, + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.ErrorScanningFile.Title" ), + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.ErrorScanningFile.Message" ), e ); + } catch ( InterruptedException e ) { + new ErrorDialog( shell, + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.ErrorScanningFile.Title" ), + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.ErrorScanningFile.Message" ), e ); + } + + return message; + } + + private String doScan( IProgressMonitor monitor ) throws KettleException { + if ( samples > 0 ) { + monitor.beginTask( + BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile" ), samples + 1 ); + } else { + monitor.beginTask( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Task.ScanningFile" ), 2 ); + } + + String line = ""; + long fileLineNumber = 0; + + DecimalFormatSymbols dfs = new DecimalFormatSymbols(); + + int nrfields = meta.getInputFields().length; + + RowMetaInterface outputRowMeta = new RowMeta(); + meta.getFields( outputRowMeta, null, null, null, transMeta, null, null ); + + // Remove the storage meta-data (don't go for lazy conversion during scan) + for ( ValueMetaInterface valueMeta : outputRowMeta.getValueMetaList() ) { + valueMeta.setStorageMetadata( null ); + valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); + } + + RowMetaInterface convertRowMeta = outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING ); + + // How many null values? + int[] nrnull = new int[nrfields]; // How many times null value? + + // String info + String[] minstr = new String[nrfields]; // min string + String[] maxstr = new String[nrfields]; // max string + boolean[] firststr = new boolean[nrfields]; // first occ. of string? + + // Date info + boolean[] isDate = new boolean[nrfields]; // is the field perhaps a Date? + int[] dateFormatCount = new int[nrfields]; // How many date formats work? + boolean[][] dateFormat = new boolean[nrfields][Const.getDateFormats().length]; // What are the date formats that + // work? + Date[][] minDate = new Date[nrfields][Const.getDateFormats().length]; // min date value + Date[][] maxDate = new Date[nrfields][Const.getDateFormats().length]; // max date value + + // Number info + boolean[] isNumber = new boolean[nrfields]; // is the field perhaps a Number? + int[] numberFormatCount = new int[nrfields]; // How many number formats work? + boolean[][] numberFormat = new boolean[nrfields][Const.getNumberFormats().length]; // What are the number format + // that work? + double[][] minValue = new double[nrfields][Const.getDateFormats().length]; // min number value + double[][] maxValue = new double[nrfields][Const.getDateFormats().length]; // max number value + int[][] numberPrecision = new int[nrfields][Const.getNumberFormats().length]; // remember the precision? + int[][] numberLength = new int[nrfields][Const.getNumberFormats().length]; // remember the length? + + for ( int i = 0; i < nrfields; i++ ) { + TextFileInputField field = meta.getInputFields()[i]; + + if ( log.isDebug() ) { + debug = "init field #" + i; + } + + if ( replaceMeta ) { // Clear previous info... + + field.setName( meta.getInputFields()[i].getName() ); + field.setType( meta.getInputFields()[i].getType() ); + field.setFormat( "" ); + field.setLength( -1 ); + field.setPrecision( -1 ); + field.setCurrencySymbol( dfs.getCurrencySymbol() ); + field.setDecimalSymbol( "" + dfs.getDecimalSeparator() ); + field.setGroupSymbol( "" + dfs.getGroupingSeparator() ); + field.setNullString( "-" ); + field.setTrimType( ValueMetaInterface.TRIM_TYPE_NONE ); + } + + nrnull[i] = 0; + minstr[i] = ""; + maxstr[i] = ""; + firststr[i] = true; + + // Init data guess + isDate[i] = true; + for ( int j = 0; j < Const.getDateFormats().length; j++ ) { + dateFormat[i][j] = true; + minDate[i][j] = Const.MAX_DATE; + maxDate[i][j] = Const.MIN_DATE; + } + dateFormatCount[i] = Const.getDateFormats().length; + + // Init number guess + isNumber[i] = true; + for ( int j = 0; j < Const.getNumberFormats().length; j++ ) { + numberFormat[i][j] = true; + minValue[i][j] = Double.MAX_VALUE; + maxValue[i][j] = -Double.MAX_VALUE; + numberPrecision[i][j] = -1; + numberLength[i][j] = -1; + } + numberFormatCount[i] = Const.getNumberFormats().length; + } + + InputFileMetaInterface strinfo = (InputFileMetaInterface) meta.clone(); + for ( int i = 0; i < nrfields; i++ ) { + strinfo.getInputFields()[i].setType( ValueMetaInterface.TYPE_STRING ); + } + + // Sample rows... + debug = "get first line"; + + StringBuilder lineBuffer = new StringBuilder( 256 ); + int fileFormatType = meta.getFileFormatTypeNr(); + + // If the file has a header we overwrite the first line + // However, if it doesn't have a header, take a new line + // + + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + fileLineNumber++; + int skipped = 1; + + if ( meta.hasHeader() ) { + + while ( line != null && skipped < meta.getNrHeaderLines() ) { + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + skipped++; + fileLineNumber++; + } + } + int linenr = 1; + + List evaluators = new ArrayList(); + + // Allocate number and date parsers + DecimalFormat df2 = (DecimalFormat) NumberFormat.getInstance(); + DecimalFormatSymbols dfs2 = new DecimalFormatSymbols(); + SimpleDateFormat daf2 = new SimpleDateFormat(); + + boolean errorFound = false; + while ( !errorFound && line != null && ( linenr <= samples || samples == 0 ) && !monitor.isCanceled() ) { + monitor.subTask( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Task.ScanningLine", "" + + linenr ) ); + if ( samples > 0 ) { + monitor.worked( 1 ); + } + + if ( log.isDebug() ) { + debug = "convert line #" + linenr + " to row"; + } + RowMetaInterface rowMeta = new RowMeta(); + meta.getFields( rowMeta, "stepname", null, null, transMeta, null, null ); + // Remove the storage meta-data (don't go for lazy conversion during scan) + for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) { + valueMeta.setStorageMetadata( null ); + valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); + } + + String delimiter = transMeta.environmentSubstitute( meta.getSeparator() ); + String enclosure = transMeta.environmentSubstitute( meta.getEnclosure() ); + String escapeCharacter = transMeta.environmentSubstitute( meta.getEscapeCharacter() ); + Object[] r = + OldTextFileInput.convertLineToRow( + log, new OldTextFileLine( line, fileLineNumber, null ), strinfo, null, 0, outputRowMeta, + convertRowMeta, meta.getFilePaths( transMeta )[0], rownumber, delimiter, enclosure, escapeCharacter, + null, false, false, false, false, false, false, false, false, null, null, false, null, null, null, + null, 0 ); + + if ( r == null ) { + errorFound = true; + continue; + } + rownumber++; + for ( int i = 0; i < nrfields && i < r.length; i++ ) { + StringEvaluator evaluator; + if ( i >= evaluators.size() ) { + evaluator = new StringEvaluator( true ); + evaluators.add( evaluator ); + } else { + evaluator = evaluators.get( i ); + } + + String string = rowMeta.getString( r, i ); + + if ( i == 0 ) { + System.out.println(); + } + evaluator.evaluateString( string ); + } + + fileLineNumber++; + if ( r != null ) { + linenr++; + } + + // Grab another line... + // + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + } + + monitor.worked( 1 ); + monitor.setTaskName( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Task.AnalyzingResults" ) ); + + // Show information on items using a dialog box + // + StringBuilder message = new StringBuilder(); + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.ResultAfterScanning", "" + + ( linenr - 1 ) ) ); + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.HorizontalLine" ) ); + + for ( int i = 0; i < nrfields; i++ ) { + TextFileInputField field = meta.getInputFields()[i]; + StringEvaluator evaluator = evaluators.get( i ); + List evaluationResults = evaluator.getStringEvaluationResults(); + + // If we didn't find any matching result, it's a String... + // + StringEvaluationResult result = evaluator.getAdvicedResult(); + if ( evaluationResults.isEmpty() ) { + field.setType( ValueMetaInterface.TYPE_STRING ); + field.setLength( evaluator.getMaxLength() ); + } + if ( result != null ) { + // Take the first option we find, list the others below... + // + ValueMetaInterface conversionMeta = result.getConversionMeta(); + field.setType( conversionMeta.getType() ); + field.setTrimType( conversionMeta.getTrimType() ); + field.setFormat( conversionMeta.getConversionMask() ); + field.setDecimalSymbol( conversionMeta.getDecimalSymbol() ); + field.setGroupSymbol( conversionMeta.getGroupingSymbol() ); + field.setLength( conversionMeta.getLength() ); + field.setPrecision( conversionMeta.getPrecision() ); + + nrnull[i] = result.getNrNull(); + minstr[i] = result.getMin() == null ? "" : result.getMin().toString(); + maxstr[i] = result.getMax() == null ? "" : result.getMax().toString(); + } + + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.FieldNumber", "" + + ( i + 1 ) ) ); + + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.FieldName", field + .getName() ) ); + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.FieldType", field + .getTypeDesc() ) ); + + switch ( field.getType() ) { + case ValueMetaInterface.TYPE_NUMBER: + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.EstimatedLength", ( field.getLength() < 0 ? "-" : "" + + field.getLength() ) ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.EstimatedPrecision", field.getPrecision() < 0 ? "-" : "" + + field.getPrecision() ) ); + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat", field + .getFormat() ) ); + + if ( !evaluationResults.isEmpty() ) { + if ( evaluationResults.size() > 1 ) { + message.append( BaseMessages + .getString( PKG, "TextFileCSVImportProgressDialog.Info.WarnNumberFormat" ) ); + } + + for ( StringEvaluationResult seResult : evaluationResults ) { + String mask = seResult.getConversionMeta().getConversionMask(); + + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat2", mask ) ); + message.append( BaseMessages + .getString( PKG, "TextFileCSVImportProgressDialog.Info.TrimType", seResult + .getConversionMeta().getTrimType() ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.NumberMinValue", seResult.getMin() ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.NumberMaxValue", seResult.getMax() ) ); + + try { + df2.applyPattern( mask ); + df2.setDecimalFormatSymbols( dfs2 ); + double mn = df2.parse( seResult.getMin().toString() ).doubleValue(); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.NumberExample", mask, seResult.getMin(), Double + .toString( mn ) ) ); + } catch ( Exception e ) { + if ( log.isDetailed() ) { + log.logDetailed( "This is unexpected: parsing [" + + seResult.getMin() + "] with format [" + mask + "] did not work." ); + } + } + } + } + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.NumberNrNullValues", "" + nrnull[i] ) ); + break; + case ValueMetaInterface.TYPE_STRING: + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.StringMaxLength", "" + + field.getLength() ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.StringMinValue", minstr[i] ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.StringMaxValue", maxstr[i] ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.StringNrNullValues", "" + nrnull[i] ) ); + break; + case ValueMetaInterface.TYPE_DATE: + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.DateMaxLength", field + .getLength() < 0 ? "-" : "" + field.getLength() ) ); + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.DateFormat", field + .getFormat() ) ); + if ( dateFormatCount[i] > 1 ) { + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.WarnDateFormat" ) ); + } + if ( !Const.isEmpty( minstr[i] ) ) { + for ( int x = 0; x < Const.getDateFormats().length; x++ ) { + if ( dateFormat[i][x] ) { + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.DateFormat2", Const.getDateFormats()[x] ) ); + Date mindate = minDate[i][x]; + Date maxdate = maxDate[i][x]; + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.DateMinValue", mindate.toString() ) ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.DateMaxValue", maxdate.toString() ) ); + + daf2.applyPattern( Const.getDateFormats()[x] ); + try { + Date md = daf2.parse( minstr[i] ); + message.append( BaseMessages.getString( + PKG, "TextFileCSVImportProgressDialog.Info.DateExample", Const.getDateFormats()[x], + minstr[i], md.toString() ) ); + } catch ( Exception e ) { + if ( log.isDetailed() ) { + log.logDetailed( "This is unexpected: parsing [" + + minstr[i] + "] with format [" + Const.getDateFormats()[x] + "] did not work." ); + } + } + } + } + } + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.DateNrNullValues", "" + + nrnull[i] ) ); + break; + default: + break; + } + if ( nrnull[i] == linenr - 1 ) { + message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.AllNullValues" ) ); + } + message.append( Const.CR ); + + } + + monitor.worked( 1 ); + monitor.done(); + + return message.toString(); + + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizard.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizard.java new file mode 100644 index 000000000000..d35eee681538 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizard.java @@ -0,0 +1,154 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import org.eclipse.jface.dialogs.IDialogSettings; +import org.eclipse.jface.wizard.IWizard; +import org.eclipse.jface.wizard.IWizardContainer; +import org.eclipse.jface.wizard.IWizardPage; +import org.eclipse.swt.graphics.Image; +import org.eclipse.swt.graphics.RGB; +import org.eclipse.swt.widgets.Composite; + +/* + * Created on 17-apr-04 + * + * To change the template for this generated file go to + * Window>Preferences>Java>Code Generation>Code and Comments + */ + +/** + * @author Matt + * + * To change the template for this generated type comment go to Window>Preferences>Java>Code + * Generation>Code and Comments + */ +public class OldTextFileImportWizard implements IWizard { + public OldTextFileImportWizard() { + super(); + } + + // Adds any last-minute pages to this wizard. + public void addPages() { + } + + // Returns whether this wizard could be finished without further user interaction. + public boolean canFinish() { + return false; + } + + // Creates this wizard's controls in the given parent control. + public void createPageControls( Composite pageContainer ) { + } + + // Disposes of this wizard and frees all SWT resources. + public void dispose() { + } + + // Returns the container of this wizard. + public IWizardContainer getContainer() { + return null; + } + + // Returns the default page image for this wizard. + public Image getDefaultPageImage() { + return null; + } + + // Returns the dialog settings for this wizard. + public IDialogSettings getDialogSettings() { + return null; + } + + // Returns the successor of the given page. + public IWizardPage getNextPage( IWizardPage page ) { + return null; + } + + // Returns the wizard page with the given name belonging to this wizard. + public IWizardPage getPage( String pageName ) { + return null; + } + + // Returns the number of pages in this wizard. + public int getPageCount() { + return 3; + } + + // Returns all the pages in this wizard. + public IWizardPage[] getPages() { + return null; + } + + // Returns the predecessor of the given page. + public IWizardPage getPreviousPage( IWizardPage page ) { + return null; + } + + // Returns the first page to be shown in this wizard. + public IWizardPage getStartingPage() { + return null; + } + + // Returns the title bar color for this wizard. + public RGB getTitleBarColor() { + return null; + } + + // Returns the window title string for this wizard. + public String getWindowTitle() { + return "WindowTitle"; + } + + // Returns whether help is available for this wizard. + public boolean isHelpAvailable() { + return false; + } + + // Returns whether this wizard needs Previous and Next buttons. + public boolean needsPreviousAndNextButtons() { + return true; + } + + // Returns whether this wizard needs a progress monitor. + public boolean needsProgressMonitor() { + return false; + } + + // Performs any actions appropriate in response to the user having pressed the Cancel button, or refuse if canceling + // now is not permitted. + public boolean performCancel() { + return false; + } + + // Performs any actions appropriate in response to the user having pressed the Finish button, or refuse if finishing + // now is not permitted. + public boolean performFinish() { + return false; + } + + // Sets or clears the container of this wizard. + public void setContainer( IWizardContainer wizardContainer ) { + } + +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage1.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage1.java new file mode 100644 index 000000000000..d297dcdc7579 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage1.java @@ -0,0 +1,121 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import java.util.List; +import java.util.Vector; + +import org.eclipse.jface.wizard.WizardPage; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.MouseAdapter; +import org.eclipse.swt.events.MouseEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Composite; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.gui.TextFileInputFieldInterface; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.ui.core.PropsUI; +import org.pentaho.di.ui.core.widget.OldTableDraw; + +public class OldTextFileImportWizardPage1 extends WizardPage // implements Listener +{ + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + private OldTableDraw wTable; + private FormData fdTable; + + private PropsUI props; + private List rows; + private Vector fields; + + public OldTextFileImportWizardPage1( String arg, PropsUI props, List rows, + Vector fields ) { + super( arg ); + this.props = props; + this.rows = rows; + this.fields = fields; + + setTitle( BaseMessages.getString( PKG, "TextFileImportWizardPage1.DialogTitle" ) ); + setDescription( BaseMessages.getString( PKG, "TextFileImportWizardPage1.DialogMessage" ) ); + } + + public void createControl( Composite parent ) { + // create the composite to hold the widgets + Composite composite = new Composite( parent, SWT.NONE ); + props.setLook( composite ); + + FormLayout compLayout = new FormLayout(); + compLayout.marginHeight = Const.FORM_MARGIN; + compLayout.marginWidth = Const.FORM_MARGIN; + composite.setLayout( compLayout ); + + MouseAdapter lsMouse = new MouseAdapter() { + public void mouseDown( MouseEvent e ) { + int s = getSize(); + // System.out.println("size = "+s); + setPageComplete( s > 0 ); + } + }; + + wTable = new OldTableDraw( composite, props, this, fields ); + wTable.setRows( rows ); + props.setLook( wTable ); + wTable.setFields( fields ); + fdTable = new FormData(); + fdTable.left = new FormAttachment( 0, 0 ); + fdTable.right = new FormAttachment( 100, 0 ); + fdTable.top = new FormAttachment( 0, 0 ); + fdTable.bottom = new FormAttachment( 100, 0 ); + wTable.setLayoutData( fdTable ); + wTable.addMouseListener( lsMouse ); + + // set the composite as the control for this page + setControl( composite ); + } + + public void setFields( Vector fields ) { + wTable.setFields( fields ); + } + + public Vector getFields() { + return wTable.getFields(); + } + + public boolean canFlipToNextPage() { + int size = getSize(); + if ( size > 0 ) { + setErrorMessage( null ); + return true; + } else { + setErrorMessage( BaseMessages.getString( PKG, "TextFileImportWizardPage1.ErrorMarkerNeeded" ) ); + return false; + } + } + + public int getSize() { + return wTable.getFields().size(); + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage2.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage2.java new file mode 100644 index 000000000000..f03af68c5227 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileImportWizardPage2.java @@ -0,0 +1,712 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import java.util.Vector; + +import org.eclipse.jface.wizard.WizardPage; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.events.ModifyEvent; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.List; +import org.eclipse.swt.widgets.MessageBox; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Text; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.Props; +import org.pentaho.di.core.gui.TextFileInputFieldInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.ui.core.PropsUI; + +public class OldTextFileImportWizardPage2 extends WizardPage { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + private List wFields; + private FormData fdFields; + + private List wSamples; + private FormData fdSamples; + + private Label wlFieldname; + private Text wFieldname; + private FormData fdlFieldname, fdFieldname; + + private Label wlPosition; + private Text wPosition; + private FormData fdlPosition, fdPosition; + + private Label wlLength; + private Text wLength; + private FormData fdlLength, fdLength; + + private Label wlFieldtype; + private CCombo wFieldtype; + private FormData fdlFieldtype, fdFieldtype; + + private Label wlFormat; + private Text wFormat; + private FormData fdlFormat, fdFormat; + + private Label wlTrimtype; + private CCombo wTrimtype; + private FormData fdlTrimtype, fdTrimtype; + + private Label wlPrecision; + private Text wPrecision; + private FormData fdlPrecision, fdPrecision; + + private Label wlDecimal; + private Text wDecimal; + private FormData fdlDecimal, fdDecimal; + + private Label wlGroup; + private Text wGroup; + private FormData fdlGroup, fdGroup; + + private Label wlCurrency; + private Text wCurrency; + private FormData fdlCurrency, fdCurrency; + + private Label wlIgnore; + private Button wIgnore; + private FormData fdlIgnore, fdIgnore; + + private Label wlRepeat; + private Button wRepeat; + private FormData fdlRepeat, fdRepeat; + + private Label wlNull; + private Text wNull; + private FormData fdlNull, fdNull; + + private Button wGuess, wGuessAll; + private FormData fdGuess, fdGuessAll; + + private Button wPrev, wNext; + private FormData fdPrev, fdNext; + + private PropsUI props; + private java.util.List rows; + private Vector fields; + + private Shell shell; + + public OldTextFileImportWizardPage2( String arg, PropsUI props, java.util.List rows, + Vector fields ) { + super( arg ); + this.props = props; + this.rows = rows; + this.fields = fields; + + setTitle( BaseMessages.getString( PKG, "TextFileImportWizardPage2.DialogTitle" ) ); + setDescription( "Give a name to the fields in this text file" ); + } + + public void createControl( Composite parent ) { + shell = parent.getShell(); + + int margin = Const.MARGIN; + int left = props.getMiddlePct() / 2; + int middle = props.getMiddlePct(); + int right = middle + left; + + // create the composite to hold the widgets + Composite composite = new Composite( parent, SWT.NONE ); + props.setLook( composite ); + + FormLayout compLayout = new FormLayout(); + compLayout.marginHeight = Const.FORM_MARGIN; + compLayout.marginWidth = Const.FORM_MARGIN; + composite.setLayout( compLayout ); + + wFields = new List( composite, SWT.SINGLE | SWT.BORDER | SWT.V_SCROLL | SWT.H_SCROLL ); + props.setLook( wFields ); + fdFields = new FormData(); + fdFields.top = new FormAttachment( 0, 0 ); + fdFields.left = new FormAttachment( 0, 0 ); + fdFields.right = new FormAttachment( left, 0 ); + fdFields.bottom = new FormAttachment( 100, 0 ); + wFields.setLayoutData( fdFields ); + + refreshFields(); + + // Fieldname line + wlFieldname = new Label( composite, SWT.RIGHT ); + wlFieldname.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Fieldname.Label" ) ); + props.setLook( wlFieldname ); + fdlFieldname = new FormData(); + fdlFieldname.left = new FormAttachment( wFields, 0 ); + fdlFieldname.top = new FormAttachment( 0, 0 ); + fdlFieldname.right = new FormAttachment( middle, 0 ); + wlFieldname.setLayoutData( fdlFieldname ); + wFieldname = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wFieldname ); + fdFieldname = new FormData(); + fdFieldname.left = new FormAttachment( middle, margin ); + fdFieldname.right = new FormAttachment( right, 0 ); + fdFieldname.top = new FormAttachment( 0, 0 ); + wFieldname.setLayoutData( fdFieldname ); + + // Position line + wlPosition = new Label( composite, SWT.RIGHT ); + wlPosition.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.FieldPosition.Label" ) ); + props.setLook( wlPosition ); + fdlPosition = new FormData(); + fdlPosition.left = new FormAttachment( wFields, 0 ); + fdlPosition.top = new FormAttachment( wFieldname, margin ); + fdlPosition.right = new FormAttachment( middle, 0 ); + wlPosition.setLayoutData( fdlPosition ); + wPosition = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wPosition ); + fdPosition = new FormData(); + fdPosition.left = new FormAttachment( middle, margin ); + fdPosition.top = new FormAttachment( wFieldname, margin ); + fdPosition.right = new FormAttachment( right, 0 ); + wPosition.setLayoutData( fdPosition ); + wPosition.setEnabled( false ); + + // Topos line + wlLength = new Label( composite, SWT.RIGHT ); + wlLength.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.FieldLength.Label" ) ); + props.setLook( wlLength ); + fdlLength = new FormData(); + fdlLength.left = new FormAttachment( wFields, 0 ); + fdlLength.top = new FormAttachment( wPosition, margin ); + fdlLength.right = new FormAttachment( middle, 0 ); + wlLength.setLayoutData( fdlLength ); + wLength = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wLength ); + fdLength = new FormData(); + fdLength.left = new FormAttachment( middle, margin ); + fdLength.top = new FormAttachment( wPosition, margin ); + fdLength.right = new FormAttachment( right, 0 ); + wLength.setLayoutData( fdLength ); + wLength.setEnabled( false ); + + // Fieldtype line + wlFieldtype = new Label( composite, SWT.RIGHT ); + wlFieldtype.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.FieldType.Label" ) ); + props.setLook( wlFieldtype ); + fdlFieldtype = new FormData(); + fdlFieldtype.left = new FormAttachment( wFields, 0 ); + fdlFieldtype.top = new FormAttachment( wLength, margin ); + fdlFieldtype.right = new FormAttachment( middle, 0 ); + wlFieldtype.setLayoutData( fdlFieldtype ); + wFieldtype = new CCombo( composite, SWT.BORDER | SWT.READ_ONLY ); + props.setLook( wFieldtype ); + for ( int i = 0; i < ValueMeta.getTypes().length; i++ ) { + wFieldtype.add( ValueMeta.getTypes()[i] ); + } + fdFieldtype = new FormData(); + fdFieldtype.left = new FormAttachment( middle, margin ); + fdFieldtype.top = new FormAttachment( wLength, margin ); + fdFieldtype.right = new FormAttachment( right, 0 ); + wFieldtype.setLayoutData( fdFieldtype ); + + // Format line + wlFormat = new Label( composite, SWT.RIGHT ); + wlFormat.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.FieldFormatter.Label" ) ); + props.setLook( wlFormat ); + fdlFormat = new FormData(); + fdlFormat.left = new FormAttachment( wFields, 0 ); + fdlFormat.top = new FormAttachment( wFieldtype, margin ); + fdlFormat.right = new FormAttachment( middle, 0 ); + wlFormat.setLayoutData( fdlFormat ); + wFormat = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wFormat ); + fdFormat = new FormData(); + fdFormat.left = new FormAttachment( middle, margin ); + fdFormat.top = new FormAttachment( wFieldtype, margin ); + fdFormat.right = new FormAttachment( right, 0 ); + wFormat.setLayoutData( fdFormat ); + + // Ignore checkbox + wlIgnore = new Label( composite, SWT.RIGHT ); + wlIgnore.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Ignore.Label" ) ); + props.setLook( wlIgnore ); + fdlIgnore = new FormData(); + fdlIgnore.left = new FormAttachment( wFields, 0 ); + fdlIgnore.top = new FormAttachment( wFormat, margin ); + fdlIgnore.right = new FormAttachment( middle, 0 ); + wlIgnore.setLayoutData( fdlIgnore ); + wIgnore = new Button( composite, SWT.CHECK ); + props.setLook( wIgnore ); + fdIgnore = new FormData(); + fdIgnore.left = new FormAttachment( middle, margin ); + fdIgnore.top = new FormAttachment( wFormat, margin ); + fdIgnore.right = new FormAttachment( right, 0 ); + wIgnore.setLayoutData( fdIgnore ); + + // Trimtype line + wlTrimtype = new Label( composite, SWT.RIGHT ); + wlTrimtype.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.TrimType.Label" ) ); + props.setLook( wlTrimtype ); + fdlTrimtype = new FormData(); + fdlTrimtype.left = new FormAttachment( wFields, 0 ); + fdlTrimtype.top = new FormAttachment( wIgnore, margin ); + fdlTrimtype.right = new FormAttachment( middle, 0 ); + wlTrimtype.setLayoutData( fdlTrimtype ); + wTrimtype = new CCombo( composite, SWT.BORDER | SWT.READ_ONLY ); + props.setLook( wTrimtype ); + for ( int i = 0; i < ValueMeta.trimTypeDesc.length; i++ ) { + wTrimtype.add( ValueMeta.trimTypeDesc[i] ); + } + fdTrimtype = new FormData(); + fdTrimtype.left = new FormAttachment( middle, margin ); + fdTrimtype.top = new FormAttachment( wIgnore, margin ); + fdTrimtype.right = new FormAttachment( right, 0 ); + wTrimtype.setLayoutData( fdTrimtype ); + + // Precision line + wlPrecision = new Label( composite, SWT.RIGHT ); + wlPrecision.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Precision.Label" ) ); + props.setLook( wlPrecision ); + fdlPrecision = new FormData(); + fdlPrecision.left = new FormAttachment( wFields, 0 ); + fdlPrecision.top = new FormAttachment( wTrimtype, margin ); + fdlPrecision.right = new FormAttachment( middle, 0 ); + wlPrecision.setLayoutData( fdlPrecision ); + wPrecision = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wPrecision ); + fdPrecision = new FormData(); + fdPrecision.left = new FormAttachment( middle, margin ); + fdPrecision.top = new FormAttachment( wTrimtype, margin ); + fdPrecision.right = new FormAttachment( right, 0 ); + wPrecision.setLayoutData( fdPrecision ); + + // Currency line + wlCurrency = new Label( composite, SWT.RIGHT ); + wlCurrency.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Currency.Label" ) ); + props.setLook( wlCurrency ); + fdlCurrency = new FormData(); + fdlCurrency.left = new FormAttachment( wFields, 0 ); + fdlCurrency.top = new FormAttachment( wPrecision, margin ); + fdlCurrency.right = new FormAttachment( middle, 0 ); + wlCurrency.setLayoutData( fdlCurrency ); + wCurrency = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wCurrency ); + fdCurrency = new FormData(); + fdCurrency.left = new FormAttachment( middle, margin ); + fdCurrency.top = new FormAttachment( wPrecision, margin ); + fdCurrency.right = new FormAttachment( right, 0 ); + wCurrency.setLayoutData( fdCurrency ); + + // Decimal line + wlDecimal = new Label( composite, SWT.RIGHT ); + wlDecimal.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Decimal.Label" ) ); + props.setLook( wlDecimal ); + fdlDecimal = new FormData(); + fdlDecimal.left = new FormAttachment( wFields, 0 ); + fdlDecimal.top = new FormAttachment( wCurrency, margin ); + fdlDecimal.right = new FormAttachment( middle, 0 ); + wlDecimal.setLayoutData( fdlDecimal ); + wDecimal = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wDecimal ); + fdDecimal = new FormData(); + fdDecimal.left = new FormAttachment( middle, margin ); + fdDecimal.top = new FormAttachment( wCurrency, margin ); + fdDecimal.right = new FormAttachment( right, 0 ); + wDecimal.setLayoutData( fdDecimal ); + + // Group line + wlGroup = new Label( composite, SWT.RIGHT ); + wlGroup.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Group.Label" ) ); + props.setLook( wlGroup ); + fdlGroup = new FormData(); + fdlGroup.left = new FormAttachment( wFields, 0 ); + fdlGroup.top = new FormAttachment( wDecimal, margin ); + fdlGroup.right = new FormAttachment( middle, 0 ); + wlGroup.setLayoutData( fdlGroup ); + wGroup = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wGroup ); + fdGroup = new FormData(); + fdGroup.left = new FormAttachment( middle, margin ); + fdGroup.top = new FormAttachment( wDecimal, margin ); + fdGroup.right = new FormAttachment( right, 0 ); + wGroup.setLayoutData( fdGroup ); + + // Ignore checkbox + wlRepeat = new Label( composite, SWT.RIGHT ); + wlRepeat.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Repeat.Label" ) ); + props.setLook( wlRepeat ); + fdlRepeat = new FormData(); + fdlRepeat.left = new FormAttachment( wFields, 0 ); + fdlRepeat.top = new FormAttachment( wGroup, margin ); + fdlRepeat.right = new FormAttachment( middle, 0 ); + wlRepeat.setLayoutData( fdlRepeat ); + wRepeat = new Button( composite, SWT.CHECK ); + props.setLook( wRepeat ); + fdRepeat = new FormData(); + fdRepeat.left = new FormAttachment( middle, margin ); + fdRepeat.top = new FormAttachment( wGroup, margin ); + fdRepeat.right = new FormAttachment( right, 0 ); + wRepeat.setLayoutData( fdRepeat ); + + // Null line + wlNull = new Label( composite, SWT.RIGHT ); + wlNull.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Null.Label" ) ); + props.setLook( wlNull ); + fdlNull = new FormData(); + fdlNull.left = new FormAttachment( wFields, 0 ); + fdlNull.top = new FormAttachment( wRepeat, margin ); + fdlNull.right = new FormAttachment( middle, 0 ); + wlNull.setLayoutData( fdlNull ); + wNull = new Text( composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wNull ); + fdNull = new FormData(); + fdNull.left = new FormAttachment( middle, margin ); + fdNull.top = new FormAttachment( wRepeat, margin ); + fdNull.right = new FormAttachment( right, 0 ); + wNull.setLayoutData( fdNull ); + + // The buttons + wPrev = new Button( composite, SWT.PUSH ); + wPrev.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Previous.Button" ) ); + fdPrev = new FormData(); + fdPrev.left = new FormAttachment( left, margin ); + fdPrev.bottom = new FormAttachment( 100, 0 ); + wPrev.setLayoutData( fdPrev ); + + // Guess button + wGuess = new Button( composite, SWT.PUSH ); + wGuess.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Guess.Button" ) ); + fdGuess = new FormData(); + fdGuess.left = new FormAttachment( wPrev, margin ); + fdGuess.bottom = new FormAttachment( 100, 0 ); + wGuess.setLayoutData( fdGuess ); + + // GuessAll button + wGuessAll = new Button( composite, SWT.PUSH ); + wGuessAll.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.GuessAll.Button" ) ); + fdGuessAll = new FormData(); + fdGuessAll.left = new FormAttachment( wGuess, 0 ); + fdGuessAll.bottom = new FormAttachment( 100, 0 ); + wGuessAll.setLayoutData( fdGuessAll ); + + wNext = new Button( composite, SWT.PUSH ); + wNext.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.Next.Button" ) ); + fdNext = new FormData(); + fdNext.left = new FormAttachment( wGuessAll, 0 ); + fdNext.bottom = new FormAttachment( 100, 0 ); + wNext.setLayoutData( fdNext ); + + // Sample list on the right... + wSamples = new List( composite, SWT.SINGLE | SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL ); + props.setLook( wSamples, Props.WIDGET_STYLE_FIXED ); + fdSamples = new FormData(); + fdSamples.top = new FormAttachment( 0, 0 ); + fdSamples.left = new FormAttachment( right, 0 ); + fdSamples.right = new FormAttachment( 100, 0 ); + fdSamples.bottom = new FormAttachment( 100, 0 ); + wSamples.setLayoutData( fdSamples ); + + wFields.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + showInfo(); + } + } ); + + if ( wFields.getItemCount() > 0 ) { + wFields.select( 0 ); + showInfo(); + } + + wFieldtype.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + int valtype = ValueMeta.getType( wFieldtype.getText() ); + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setType( valtype ); + } + } + } ); + + wFieldname.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setName( wFieldname.getText() ); + wFields.setItem( idx, wFieldname.getText() ); + } + } + } ); + + wFormat.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setFormat( wFormat.getText() ); + } + } + } ); + + wNext.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 && idx < wFields.getItemCount() - 1 ) { + wFields.select( idx + 1 ); + wFields.showSelection(); + showInfo(); + } + } + } ); + + wPrev.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx > 0 ) { + wFields.select( idx - 1 ); + wFields.showSelection(); + showInfo(); + } + } + } ); + + wIgnore.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.flipIgnored(); + } + } + } ); + + wGuess.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setSamples( wSamples.getItems() ); + field.guess(); + showInfo(); + } + } + } ); + + wGuessAll.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + MessageBox mb = new MessageBox( shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION ); + mb.setMessage( BaseMessages.getString( + PKG, "TextFileImportWizardPage2.OverwriteTypeSettings.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "TextFileImportWizardPage2.OverwriteTypeSettings.DialogTitle" ) ); + int answer = mb.open(); + if ( answer == SWT.YES ) { + for ( int i = 0; i < fields.size(); i++ ) { + TextFileInputField field = (TextFileInputField) fields.get( i ); + field.setSamples( getRowSamples( field.getPosition(), field.getLength() ) ); + field.guess(); + + wFields.select( i ); + wFields.showSelection(); + showInfo(); + } + } + + if ( wFields.getItemCount() > 0 ) { + wFields.select( 0 ); + wFields.showSelection(); + } + showInfo(); + } + } ); + + wRepeat.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.flipRepeated(); + } + } + } ); + + wCurrency.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setCurrencySymbol( wCurrency.getText() ); + } + } + } ); + + wGroup.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setGroupSymbol( wGroup.getText() ); + } + } + } ); + + wDecimal.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setDecimalSymbol( wDecimal.getText() ); + } + } + } ); + + wNull.addModifyListener( new ModifyListener() { + public void modifyText( ModifyEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setNullString( wNull.getText() ); + } + } + } ); + + wTrimtype.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + int trimType = ValueMeta.getTrimTypeByDesc( wTrimtype.getText() ); + TextFileInputField field = (TextFileInputField) fields.get( idx ); + field.setTrimType( trimType ); + } + } + } ); + + // set the composite as the control for this page + setControl( composite ); + } + + public boolean canFlipToNextPage() { + refreshFields(); + if ( wFields.getItemCount() > 0 ) { + wFields.select( 0 ); + showInfo(); + } + return false; + } + + private void refreshFields() { + wFields.removeAll(); + for ( int i = 0; i < fields.size(); i++ ) { + wFields.add( ( (TextFileInputField) fields.get( i ) ).getName() ); + } + } + + private String[] getRowSamples( int position, int length ) { + if ( position < 0 || position + length < 0 ) { + return new String[0]; + } + + String[] retval = new String[rows.size()]; + + for ( int i = 0; i < rows.size(); i++ ) { + String line = rows.get( i ); + + if ( position < line.length() ) { + if ( position + length >= line.length() ) { + retval[i] = line.substring( position ); + } else { + retval[i] = line.substring( position, position + length ); + } + } else { + retval[i] = ""; + } + } + + return retval; + } + + private void showInfo() { + int idx = wFields.getSelectionIndex(); + if ( idx >= 0 ) { + TextFileInputField field = (TextFileInputField) fields.get( idx ); + + String name = field.getName(); + int from = field.getPosition(); + int length = field.getLength(); + String type = field.getTypeDesc(); + boolean ignore = field.isIgnored(); + String format = field.getFormat(); + String trimtype = field.getTrimTypeDesc(); + int precision = field.getPrecision(); + String currency = field.getCurrencySymbol(); + String decimal = field.getDecimalSymbol(); + String group = field.getGroupSymbol(); + boolean repeat = field.isRepeated(); + String nullif = field.getNullString(); + + if ( name != null ) { + wFieldname.setText( name ); + } + wPosition.setText( "" + from ); + wLength.setText( "" + length ); + if ( type != null ) { + wFieldtype.setText( type ); + } + wIgnore.setSelection( ignore ); + if ( format != null ) { + wFormat.setText( format ); + } + if ( trimtype != null ) { + wTrimtype.setText( trimtype ); + } + wPrecision.setText( "" + precision ); + if ( currency != null ) { + wCurrency.setText( currency ); + } + if ( decimal != null ) { + wDecimal.setText( decimal ); + } + if ( group != null ) { + wGroup.setText( group ); + } + wRepeat.setSelection( repeat ); + if ( nullif != null ) { + wNull.setText( nullif ); + } + + // Clear the sample list... + wSamples.removeAll(); + String[] samples = getRowSamples( from, length ); + for ( int i = 0; i < samples.length; i++ ) { + wSamples.add( samples[i] ); + } + + } + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileInputDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileInputDialog.java new file mode 100644 index 000000000000..5baefccef4e8 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldTextFileInputDialog.java @@ -0,0 +1,3263 @@ +//CHECKSTYLE:FileLength:OFF +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Vector; + +import org.apache.commons.vfs2.FileObject; +import org.eclipse.jface.wizard.Wizard; +import org.eclipse.jface.wizard.WizardDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.custom.ScrolledComposite; +import org.eclipse.swt.events.FocusListener; +import org.eclipse.swt.events.ModifyEvent; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.ShellAdapter; +import org.eclipse.swt.events.ShellEvent; +import org.eclipse.swt.graphics.Cursor; +import org.eclipse.swt.graphics.Rectangle; +import org.eclipse.swt.layout.FillLayout; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.DirectoryDialog; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Event; +import org.eclipse.swt.widgets.FileDialog; +import org.eclipse.swt.widgets.Group; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Listener; +import org.eclipse.swt.widgets.MessageBox; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Table; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; +import org.pentaho.di.core.Const; +import org.pentaho.di.core.Props; +import org.pentaho.di.core.compress.CompressionInputStream; +import org.pentaho.di.core.compress.CompressionProvider; +import org.pentaho.di.core.compress.CompressionProviderFactory; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.fileinput.FileInputList; +import org.pentaho.di.core.gui.TextFileInputFieldInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.util.EnvUtil; +import org.pentaho.di.core.vfs.KettleVFS; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.TransPreviewFactory; +import org.pentaho.di.trans.step.BaseStepMeta; +import org.pentaho.di.trans.step.StepDialogInterface; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.oldtextfileinput.OldEncodingType; +import org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileFilter; +import org.pentaho.di.trans.steps.oldtextfileinput.OldTextFileInput; +import org.pentaho.di.ui.core.dialog.EnterNumberDialog; +import org.pentaho.di.ui.core.dialog.EnterSelectionDialog; +import org.pentaho.di.ui.core.dialog.EnterTextDialog; +import org.pentaho.di.ui.core.dialog.ErrorDialog; +import org.pentaho.di.ui.core.dialog.PreviewRowsDialog; +import org.pentaho.di.ui.core.gui.GUIResource; +import org.pentaho.di.ui.core.widget.ColumnInfo; +import org.pentaho.di.ui.core.widget.TableView; +import org.pentaho.di.ui.core.widget.TextVar; +import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog; +import org.pentaho.di.ui.trans.step.BaseStepDialog; + +public class OldTextFileInputDialog extends BaseStepDialog implements StepDialogInterface { + private static Class PKG = OldTextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + private static final String[] YES_NO_COMBO = new String[] { BaseMessages.getString( PKG, "System.Combo.No" ), + BaseMessages.getString( PKG, "System.Combo.Yes" ) }; + + private CTabFolder wTabFolder; + private FormData fdTabFolder; + + private CTabItem wFileTab; + private CTabItem wContentTab; + private CTabItem wErrorTab; + private CTabItem wFilterTab; + private CTabItem wFieldsTab; + + private ScrolledComposite wFileSComp; + private ScrolledComposite wContentSComp; + private ScrolledComposite wErrorSComp; + + private Composite wFileComp; + private Composite wContentComp; + private Composite wErrorComp; + private Composite wFilterComp; + private Composite wFieldsComp; + + private FormData fdFileComp; + private FormData fdContentComp; + private FormData fdErrorComp; + private FormData fdFilterComp; + private FormData fdFieldsComp; + + private Group gAccepting; + private FormData fdAccepting; + + private Label wlExcludeFilemask; + private TextVar wExcludeFilemask; + private FormData fdlExcludeFilemask, fdExcludeFilemask; + + private Label wlAccFilenames; + private Button wAccFilenames; + private FormData fdlAccFilenames, fdAccFilenames; + + private Label wlPassThruFields; + private Button wPassThruFields; + private FormData fdlPassThruFields, fdPassThruFields; + + private Label wlAccField; + private Text wAccField; + private FormData fdlAccField, fdAccField; + + private Label wlAccStep; + private CCombo wAccStep; + private FormData fdlAccStep, fdAccStep; + + private Label wlFilename; + private Button wbbFilename; // Browse: add file or directory + private Button wbdFilename; // Delete + private Button wbeFilename; // Edit + private Button wbaFilename; // Add or change + private TextVar wFilename; + private FormData fdlFilename, fdbFilename, fdbdFilename, fdbeFilename, fdbaFilename, fdFilename; + + private Label wlFilenameList; + private TableView wFilenameList; + private FormData fdlFilenameList, fdFilenameList; + + private Label wlFilemask; + // PDI-8664 + private TextVar wFilemask; + private FormData fdlFilemask, fdFilemask; + + private Button wbShowFiles; + private FormData fdbShowFiles; + + private Button wFirst; + private FormData fdFirst; + private Listener lsFirst; + + private Button wFirstHeader; + private FormData fdFirstHeader; + private Listener lsFirstHeader; + + private Label wlFiletype; + private CCombo wFiletype; + private FormData fdlFiletype, fdFiletype; + + private Label wlSeparator; + private Button wbSeparator; + private TextVar wSeparator; + private FormData fdlSeparator, fdbSeparator, fdSeparator; + + private Label wlEnclosure; + private Text wEnclosure; + private FormData fdlEnclosure, fdEnclosure; + + private Label wlEnclBreaks; + private Button wEnclBreaks; + private FormData fdlEnclBreaks, fdEnclBreaks; + + private Label wlEscape; + private Text wEscape; + private FormData fdlEscape, fdEscape; + + private Label wlHeader; + private Button wHeader; + private FormData fdlHeader, fdHeader; + + private Label wlNrHeader; + private Text wNrHeader; + private FormData fdlNrHeader, fdNrHeader; + + private Label wlFooter; + private Button wFooter; + private FormData fdlFooter, fdFooter; + + private Label wlNrFooter; + private Text wNrFooter; + private FormData fdlNrFooter, fdNrFooter; + + private Label wlWraps; + private Button wWraps; + private FormData fdlWraps, fdWraps; + + private Label wlNrWraps; + private Text wNrWraps; + private FormData fdlNrWraps, fdNrWraps; + + private Label wlLayoutPaged; + private Button wLayoutPaged; + private FormData fdlLayoutPaged, fdLayoutPaged; + + private Label wlNrLinesPerPage; + private Text wNrLinesPerPage; + private FormData fdlNrLinesPerPage, fdNrLinesPerPage; + + private Label wlNrLinesDocHeader; + private Text wNrLinesDocHeader; + private FormData fdlNrLinesDocHeader, fdNrLinesDocHeader; + + private Label wlCompression; + private CCombo wCompression; + private FormData fdlCompression, fdCompression; + + private Label wlNoempty; + private Button wNoempty; + private FormData fdlNoempty, fdNoempty; + + private Label wlInclFilename; + private Button wInclFilename; + private FormData fdlInclFilename, fdInclFilename; + + private Label wlInclFilenameField; + private Text wInclFilenameField; + private FormData fdlInclFilenameField, fdInclFilenameField; + + private Label wlInclRownum; + private Button wInclRownum; + private FormData fdlInclRownum, fdRownum; + + private Label wlRownumByFileField; + private Button wRownumByFile; + private FormData fdlRownumByFile, fdRownumByFile; + + private Label wlInclRownumField; + private Text wInclRownumField; + private FormData fdlInclRownumField, fdInclRownumField; + + private Label wlFormat; + private CCombo wFormat; + private FormData fdlFormat, fdFormat; + + private Label wlEncoding; + private CCombo wEncoding; + private FormData fdlEncoding, fdEncoding; + + private Label wlLimit; + private Text wLimit; + private FormData fdlLimit, fdLimit; + + private Label wlDateLenient; + private Button wDateLenient; + private FormData fdlDateLenient, fdDateLenient; + + private Label wlDateLocale; + private CCombo wDateLocale; + private FormData fdlDateLocale, fdDateLocale; + + // ERROR HANDLING... + private Label wlErrorIgnored; + private Button wErrorIgnored; + private FormData fdlErrorIgnored, fdErrorIgnored; + + private Label wlSkipBadFiles; + private Button wSkipBadFiles; + private FormData fdlSkipBadFiles, fdSkipBadFiles; + + private Label wlSkipErrorLines; + private Button wSkipErrorLines; + private FormData fdlSkipErrorLines, fdSkipErrorLines; + + private Label wlErrorCount; + private Text wErrorCount; + private FormData fdlErrorCount, fdErrorCount; + + private Label wlErrorFields; + private Text wErrorFields; + private FormData fdlErrorFields, fdErrorFields; + + private Label wlErrorText; + private Text wErrorText; + private FormData fdlErrorText, fdErrorText; + + // New entries for intelligent error handling AKA replay functionality + // Bad files destination directory + private Label wlWarnDestDir; + private Button wbbWarnDestDir; // Browse: add file or directory + private TextVar wWarnDestDir; + private FormData fdlWarnDestDir, fdbBadDestDir, fdBadDestDir; + private Label wlWarnExt; + private Text wWarnExt; + private FormData fdlWarnDestExt, fdWarnDestExt; + + // Error messages files destination directory + private Label wlErrorDestDir; + private Button wbbErrorDestDir; // Browse: add file or directory + private TextVar wErrorDestDir; + private FormData fdlErrorDestDir, fdbErrorDestDir, fdErrorDestDir; + private Label wlErrorExt; + private Text wErrorExt; + private FormData fdlErrorDestExt, fdErrorDestExt; + + // Line numbers files destination directory + private Label wlLineNrDestDir; + private Button wbbLineNrDestDir; // Browse: add file or directory + private TextVar wLineNrDestDir; + private FormData fdlLineNrDestDir, fdbLineNrDestDir, fdLineNrDestDir; + private Label wlLineNrExt; + private Text wLineNrExt; + private FormData fdlLineNrDestExt, fdLineNrDestExt; + + private TableView wFilter; + private FormData fdFilter; + + private TableView wFields; + private FormData fdFields; + + private FormData fdlAddResult, fdAddFileResult, fdAddResult; + + private Group wAddFileResult; + + private Label wlAddResult; + + private Button wAddResult; + + private OldTextFileInputMeta input; + + // Wizard info... + private Vector fields; + + private String[] dateLocale; + + private int middle, margin; + private ModifyListener lsMod; + + public static final int[] dateLengths = new int[] { 23, 19, 14, 10, 10, 10, 10, 8, 8, 8, 8, 6, 6 }; + + private boolean gotEncodings = false; + + protected boolean firstClickOnDateLocale; + + private CTabItem wAdditionalFieldsTab; + private Composite wAdditionalFieldsComp; + private FormData fdAdditionalFieldsComp; + + private Label wlShortFileFieldName; + private FormData fdlShortFileFieldName; + private TextVar wShortFileFieldName; + private FormData fdShortFileFieldName; + private Label wlPathFieldName; + private FormData fdlPathFieldName; + private TextVar wPathFieldName; + private FormData fdPathFieldName; + + private Label wlIsHiddenName; + private FormData fdlIsHiddenName; + private TextVar wIsHiddenName; + private FormData fdIsHiddenName; + private Label wlLastModificationTimeName; + private FormData fdlLastModificationTimeName; + private TextVar wLastModificationTimeName; + private FormData fdLastModificationTimeName; + private Label wlUriName; + private FormData fdlUriName; + private TextVar wUriName; + private FormData fdUriName; + private Label wlRootUriName; + private FormData fdlRootUriName; + private TextVar wRootUriName; + private FormData fdRootUriName; + private Label wlExtensionFieldName; + private FormData fdlExtensionFieldName; + private TextVar wExtensionFieldName; + private FormData fdExtensionFieldName; + private Label wlSizeFieldName; + private FormData fdlSizeFieldName; + private TextVar wSizeFieldName; + private FormData fdSizeFieldName; + + private Label wlBadFileField; + + private FormData fdlBadFileField; + + private Text wBadFileField; + + private FormData fdBadFileField; + + private Label wlBadFileMessageField; + + private FormData fdlBadFileMessageField; + + private Text wBadFileMessageField; + + private FormData fdBadFileMessageField; + + public OldTextFileInputDialog( Shell parent, Object in, TransMeta transMeta, String sname ) { + super( parent, (BaseStepMeta) in, transMeta, sname ); + input = (OldTextFileInputMeta) in; + firstClickOnDateLocale = true; + } + + public String open() { + Shell parent = getParent(); + Display display = parent.getDisplay(); + + shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN ); + props.setLook( shell ); + setShellImage( shell, input ); + + lsMod = new ModifyListener() { + public void modifyText( ModifyEvent e ) { + input.setChanged(); + } + }; + changed = input.hasChanged(); + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = Const.FORM_MARGIN; + formLayout.marginHeight = Const.FORM_MARGIN; + + shell.setLayout( formLayout ); + shell.setText( BaseMessages.getString( PKG, "TextFileInputDialog.DialogTitle" ) ); + + middle = props.getMiddlePct(); + margin = Const.MARGIN; + + // Stepname line + wlStepname = new Label( shell, SWT.RIGHT ); + wlStepname.setText( BaseMessages.getString( PKG, "System.Label.StepName" ) ); + props.setLook( wlStepname ); + fdlStepname = new FormData(); + fdlStepname.left = new FormAttachment( 0, 0 ); + fdlStepname.top = new FormAttachment( 0, margin ); + fdlStepname.right = new FormAttachment( middle, -margin ); + wlStepname.setLayoutData( fdlStepname ); + wStepname = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wStepname.setText( stepname ); + props.setLook( wStepname ); + wStepname.addModifyListener( lsMod ); + fdStepname = new FormData(); + fdStepname.left = new FormAttachment( middle, 0 ); + fdStepname.top = new FormAttachment( 0, margin ); + fdStepname.right = new FormAttachment( 100, 0 ); + wStepname.setLayoutData( fdStepname ); + + wTabFolder = new CTabFolder( shell, SWT.BORDER ); + props.setLook( wTabFolder, Props.WIDGET_STYLE_TAB ); + wTabFolder.setSimple( false ); + + addFilesTab(); + addContentTab(); + addErrorTab(); + addFiltersTabs(); + addFieldsTabs(); + addAdditionalFieldsTab(); + + fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment( 0, 0 ); + fdTabFolder.top = new FormAttachment( wStepname, margin ); + fdTabFolder.right = new FormAttachment( 100, 0 ); + fdTabFolder.bottom = new FormAttachment( 100, -50 ); + wTabFolder.setLayoutData( fdTabFolder ); + + wOK = new Button( shell, SWT.PUSH ); + wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) ); + + wPreview = new Button( shell, SWT.PUSH ); + wPreview.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Preview.Button" ) ); + + wCancel = new Button( shell, SWT.PUSH ); + wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) ); + + setButtonPositions( new Button[] { wOK, wPreview, wCancel }, margin, wTabFolder ); + + // Add listeners + lsOK = new Listener() { + public void handleEvent( Event e ) { + ok(); + } + }; + lsFirst = new Listener() { + public void handleEvent( Event e ) { + first( false ); + } + }; + lsFirstHeader = new Listener() { + public void handleEvent( Event e ) { + first( true ); + } + }; + lsGet = new Listener() { + public void handleEvent( Event e ) { + get(); + } + }; + lsPreview = new Listener() { + public void handleEvent( Event e ) { + preview(); + } + }; + lsCancel = new Listener() { + public void handleEvent( Event e ) { + cancel(); + } + }; + + wOK.addListener( SWT.Selection, lsOK ); + wFirst.addListener( SWT.Selection, lsFirst ); + wFirstHeader.addListener( SWT.Selection, lsFirstHeader ); + wGet.addListener( SWT.Selection, lsGet ); + wPreview.addListener( SWT.Selection, lsPreview ); + wCancel.addListener( SWT.Selection, lsCancel ); + + lsDef = new SelectionAdapter() { + public void widgetDefaultSelected( SelectionEvent e ) { + ok(); + } + }; + + wAccFilenames.addSelectionListener( lsDef ); + wStepname.addSelectionListener( lsDef ); + // wFilename.addSelectionListener( lsDef ); + wSeparator.addSelectionListener( lsDef ); + wLimit.addSelectionListener( lsDef ); + wInclRownumField.addSelectionListener( lsDef ); + wInclFilenameField.addSelectionListener( lsDef ); + wNrHeader.addSelectionListener( lsDef ); + wNrFooter.addSelectionListener( lsDef ); + wNrWraps.addSelectionListener( lsDef ); + wWarnDestDir.addSelectionListener( lsDef ); + wWarnExt.addSelectionListener( lsDef ); + wErrorDestDir.addSelectionListener( lsDef ); + wErrorExt.addSelectionListener( lsDef ); + wLineNrDestDir.addSelectionListener( lsDef ); + wLineNrExt.addSelectionListener( lsDef ); + wAccField.addSelectionListener( lsDef ); + + // Add the file to the list of files... + SelectionAdapter selA = new SelectionAdapter() { + public void widgetSelected( SelectionEvent arg0 ) { + wFilenameList.add( wFilename.getText(), wFilemask.getText(), wExcludeFilemask.getText(), + OldTextFileInputMeta.RequiredFilesCode[ 0 ], OldTextFileInputMeta.RequiredFilesCode[ 0 ] ); + wFilename.setText( "" ); + wFilemask.setText( "" ); + wExcludeFilemask.setText( "" ); + wFilenameList.removeEmptyRows(); + wFilenameList.setRowNums(); + wFilenameList.optWidth( true ); + } + }; + wbaFilename.addSelectionListener( selA ); + wFilename.addSelectionListener( selA ); + + // Delete files from the list of files... + wbdFilename.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent arg0 ) { + int[] idx = wFilenameList.getSelectionIndices(); + wFilenameList.remove( idx ); + wFilenameList.removeEmptyRows(); + wFilenameList.setRowNums(); + } + } ); + + // Edit the selected file & remove from the list... + wbeFilename.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent arg0 ) { + int idx = wFilenameList.getSelectionIndex(); + if ( idx >= 0 ) { + String[] string = wFilenameList.getItem( idx ); + wFilename.setText( string[0] ); + wFilemask.setText( string[1] ); + wExcludeFilemask.setText( string[2] ); + wFilenameList.remove( idx ); + } + wFilenameList.removeEmptyRows(); + wFilenameList.setRowNums(); + } + } ); + + // Show the files that are selected at this time... + wbShowFiles.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + showFiles(); + } + } ); + + // Allow the insertion of tabs as separator... + wbSeparator.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent se ) { + wSeparator.getTextWidget().insert( "\t" ); + } + } ); + + SelectionAdapter lsFlags = new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + setFlags(); + } + }; + + // Enable/disable the right fields... + wInclFilename.addSelectionListener( lsFlags ); + wInclRownum.addSelectionListener( lsFlags ); + wRownumByFile.addSelectionListener( lsFlags ); + wErrorIgnored.addSelectionListener( lsFlags ); + wSkipBadFiles.addSelectionListener( lsFlags ); + wHeader.addSelectionListener( lsFlags ); + wFooter.addSelectionListener( lsFlags ); + wWraps.addSelectionListener( lsFlags ); + wLayoutPaged.addSelectionListener( lsFlags ); + wAccFilenames.addSelectionListener( lsFlags ); + + // Listen to the Browse... button + wbbFilename.addSelectionListener( new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + if ( wFilemask.getText() != null && wFilemask.getText().length() > 0 ) // A mask: a directory! + { + DirectoryDialog dialog = new DirectoryDialog( shell, SWT.OPEN ); + if ( wFilename.getText() != null ) { + String fpath = transMeta.environmentSubstitute( wFilename.getText() ); + dialog.setFilterPath( fpath ); + } + + if ( dialog.open() != null ) { + String str = dialog.getFilterPath(); + wFilename.setText( str ); + } + } else { + FileDialog dialog = new FileDialog( shell, SWT.OPEN ); + CompressionProvider provider = + CompressionProviderFactory.getInstance().getCompressionProviderByName( wCompression.getText() ); + + List filterExtensions = new ArrayList(); + List filterNames = new ArrayList(); + + if ( !Const.isEmpty( provider.getDefaultExtension() ) && !Const.isEmpty( provider.getName() ) ) { + filterExtensions.add( "*." + provider.getDefaultExtension() ); + filterNames.add( provider.getName() + " files" ); + } + + filterExtensions.add( "*.txt;*.csv" ); + filterNames.add( BaseMessages.getString( PKG, "TextFileInputDialog.FileType.TextAndCSVFiles" ) ); + filterExtensions.add( "*.csv" ); + filterNames.add( BaseMessages.getString( PKG, "System.FileType.CSVFiles" ) ); + filterExtensions.add( "*.txt" ); + filterNames.add( BaseMessages.getString( PKG, "System.FileType.TextFiles" ) ); + filterExtensions.add( "*" ); + filterNames.add( BaseMessages.getString( PKG, "System.FileType.AllFiles" ) ); + dialog.setFilterExtensions( filterExtensions.toArray( new String[filterExtensions.size()] ) ); + + if ( wFilename.getText() != null ) { + String fname = transMeta.environmentSubstitute( wFilename.getText() ); + dialog.setFileName( fname ); + } + + dialog.setFilterNames( filterNames.toArray( new String[filterNames.size()] ) ); + + if ( dialog.open() != null ) { + String str = dialog.getFilterPath() + System.getProperty( "file.separator" ) + dialog.getFileName(); + wFilename.setText( str ); + } + } + } + } ); + + // Detect X or ALT-F4 or something that kills this window... + shell.addShellListener( new ShellAdapter() { + public void shellClosed( ShellEvent e ) { + cancel(); + } + } ); + + wTabFolder.setSelection( 0 ); + + // Set the shell size, based upon previous time... + getData( input ); + + setSize(); + + shell.open(); + while ( !shell.isDisposed() ) { + if ( !display.readAndDispatch() ) { + display.sleep(); + } + } + return stepname; + } + + private void showFiles() { + OldTextFileInputMeta tfii = new OldTextFileInputMeta(); + getInfo( tfii ); + String[] files = tfii.getFilePaths( transMeta ); + if ( files != null && files.length > 0 ) { + EnterSelectionDialog esd = new EnterSelectionDialog( shell, files, "Files read", "Files read:" ); + esd.setViewOnly(); + esd.open(); + } else { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.NoFilesFound.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "System.Dialog.Error.Title" ) ); + mb.open(); + } + } + + private void addFilesTab() { + // //////////////////////// + // START OF FILE TAB /// + // //////////////////////// + + wFileTab = new CTabItem( wTabFolder, SWT.NONE ); + wFileTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FileTab.TabTitle" ) ); + + wFileSComp = new ScrolledComposite( wTabFolder, SWT.V_SCROLL | SWT.H_SCROLL ); + wFileSComp.setLayout( new FillLayout() ); + + wFileComp = new Composite( wFileSComp, SWT.NONE ); + props.setLook( wFileComp ); + + FormLayout fileLayout = new FormLayout(); + fileLayout.marginWidth = 3; + fileLayout.marginHeight = 3; + wFileComp.setLayout( fileLayout ); + + // Filename line + wlFilename = new Label( wFileComp, SWT.RIGHT ); + wlFilename.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Filename.Label" ) ); + props.setLook( wlFilename ); + fdlFilename = new FormData(); + fdlFilename.left = new FormAttachment( 0, 0 ); + fdlFilename.top = new FormAttachment( 0, 0 ); + fdlFilename.right = new FormAttachment( middle, -margin ); + wlFilename.setLayoutData( fdlFilename ); + + wbbFilename = new Button( wFileComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbbFilename ); + wbbFilename.setText( BaseMessages.getString( PKG, "System.Button.Browse" ) ); + wbbFilename.setToolTipText( BaseMessages.getString( PKG, "System.Tooltip.BrowseForFileOrDirAndAdd" ) ); + fdbFilename = new FormData(); + fdbFilename.right = new FormAttachment( 100, 0 ); + fdbFilename.top = new FormAttachment( 0, 0 ); + wbbFilename.setLayoutData( fdbFilename ); + + wbaFilename = new Button( wFileComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbaFilename ); + wbaFilename.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameAdd.Button" ) ); + wbaFilename.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameAdd.Tooltip" ) ); + fdbaFilename = new FormData(); + fdbaFilename.right = new FormAttachment( wbbFilename, -margin ); + fdbaFilename.top = new FormAttachment( 0, 0 ); + wbaFilename.setLayoutData( fdbaFilename ); + + wFilename = new TextVar( transMeta, wFileComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wFilename ); + wFilename.addModifyListener( lsMod ); + fdFilename = new FormData(); + fdFilename.left = new FormAttachment( middle, 0 ); + fdFilename.right = new FormAttachment( wbaFilename, -margin ); + fdFilename.top = new FormAttachment( 0, 0 ); + wFilename.setLayoutData( fdFilename ); + + wlFilemask = new Label( wFileComp, SWT.RIGHT ); + wlFilemask.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Filemask.Label" ) ); + props.setLook( wlFilemask ); + fdlFilemask = new FormData(); + fdlFilemask.left = new FormAttachment( 0, 0 ); + fdlFilemask.top = new FormAttachment( wFilename, margin ); + fdlFilemask.right = new FormAttachment( middle, -margin ); + wlFilemask.setLayoutData( fdlFilemask ); + + // PDI-8664 + wFilemask = new TextVar( transMeta, wFileComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + + props.setLook( wFilemask ); + wFilemask.addModifyListener( lsMod ); + fdFilemask = new FormData(); + fdFilemask.left = new FormAttachment( middle, 0 ); + fdFilemask.top = new FormAttachment( wFilename, margin ); + fdFilemask.right = new FormAttachment( wbaFilename, -margin ); + wFilemask.setLayoutData( fdFilemask ); + + wlExcludeFilemask = new Label( wFileComp, SWT.RIGHT ); + wlExcludeFilemask.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ExcludeFilemask.Label" ) ); + props.setLook( wlExcludeFilemask ); + fdlExcludeFilemask = new FormData(); + fdlExcludeFilemask.left = new FormAttachment( 0, 0 ); + fdlExcludeFilemask.top = new FormAttachment( wFilemask, margin ); + fdlExcludeFilemask.right = new FormAttachment( middle, -margin ); + wlExcludeFilemask.setLayoutData( fdlExcludeFilemask ); + wExcludeFilemask = new TextVar( transMeta, wFileComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wExcludeFilemask ); + wExcludeFilemask.addModifyListener( lsMod ); + fdExcludeFilemask = new FormData(); + fdExcludeFilemask.left = new FormAttachment( middle, 0 ); + fdExcludeFilemask.top = new FormAttachment( wFilemask, margin ); + fdExcludeFilemask.right = new FormAttachment( wFilename, 0, SWT.RIGHT ); + wExcludeFilemask.setLayoutData( fdExcludeFilemask ); + + // Filename list line + wlFilenameList = new Label( wFileComp, SWT.RIGHT ); + wlFilenameList.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameList.Label" ) ); + props.setLook( wlFilenameList ); + fdlFilenameList = new FormData(); + fdlFilenameList.left = new FormAttachment( 0, 0 ); + fdlFilenameList.top = new FormAttachment( wExcludeFilemask, margin ); + fdlFilenameList.right = new FormAttachment( middle, -margin ); + wlFilenameList.setLayoutData( fdlFilenameList ); + + // Buttons to the right of the screen... + wbdFilename = new Button( wFileComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbdFilename ); + wbdFilename.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameDelete.Button" ) ); + wbdFilename.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameDelete.Tooltip" ) ); + fdbdFilename = new FormData(); + fdbdFilename.right = new FormAttachment( 100, 0 ); + fdbdFilename.top = new FormAttachment( wExcludeFilemask, 40 ); + wbdFilename.setLayoutData( fdbdFilename ); + + wbeFilename = new Button( wFileComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbeFilename ); + wbeFilename.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameEdit.Button" ) ); + wbeFilename.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.FilenameEdit.Tooltip" ) ); + fdbeFilename = new FormData(); + fdbeFilename.right = new FormAttachment( 100, 0 ); + fdbeFilename.left = new FormAttachment( wbdFilename, 0, SWT.LEFT ); + fdbeFilename.top = new FormAttachment( wbdFilename, margin ); + wbeFilename.setLayoutData( fdbeFilename ); + + wbShowFiles = new Button( wFileComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbShowFiles ); + wbShowFiles.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ShowFiles.Button" ) ); + fdbShowFiles = new FormData(); + fdbShowFiles.left = new FormAttachment( middle, 0 ); + fdbShowFiles.bottom = new FormAttachment( 100, 0 ); + wbShowFiles.setLayoutData( fdbShowFiles ); + + wFirst = new Button( wFileComp, SWT.PUSH ); + wFirst.setText( BaseMessages.getString( PKG, "TextFileInputDialog.First.Button" ) ); + fdFirst = new FormData(); + fdFirst.left = new FormAttachment( wbShowFiles, margin * 2 ); + fdFirst.bottom = new FormAttachment( 100, 0 ); + wFirst.setLayoutData( fdFirst ); + + wFirstHeader = new Button( wFileComp, SWT.PUSH ); + wFirstHeader.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FirstHeader.Button" ) ); + fdFirstHeader = new FormData(); + fdFirstHeader.left = new FormAttachment( wFirst, margin * 2 ); + fdFirstHeader.bottom = new FormAttachment( 100, 0 ); + wFirstHeader.setLayoutData( fdFirstHeader ); + + // Accepting filenames group + // + + gAccepting = new Group( wFileComp, SWT.SHADOW_ETCHED_IN ); + gAccepting.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptingGroup.Label" ) ); + FormLayout acceptingLayout = new FormLayout(); + acceptingLayout.marginWidth = 3; + acceptingLayout.marginHeight = 3; + gAccepting.setLayout( acceptingLayout ); + props.setLook( gAccepting ); + + // Accept filenames from previous steps? + // + wlAccFilenames = new Label( gAccepting, SWT.RIGHT ); + wlAccFilenames.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptFilenames.Label" ) ); + props.setLook( wlAccFilenames ); + fdlAccFilenames = new FormData(); + fdlAccFilenames.top = new FormAttachment( 0, margin ); + fdlAccFilenames.left = new FormAttachment( 0, 0 ); + fdlAccFilenames.right = new FormAttachment( middle, -margin ); + wlAccFilenames.setLayoutData( fdlAccFilenames ); + wAccFilenames = new Button( gAccepting, SWT.CHECK ); + wAccFilenames.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptFilenames.Tooltip" ) ); + props.setLook( wAccFilenames ); + fdAccFilenames = new FormData(); + fdAccFilenames.top = new FormAttachment( 0, margin ); + fdAccFilenames.left = new FormAttachment( middle, 0 ); + fdAccFilenames.right = new FormAttachment( 100, 0 ); + wAccFilenames.setLayoutData( fdAccFilenames ); + + // Accept filenames from previous steps? + // + wlPassThruFields = new Label( gAccepting, SWT.RIGHT ); + wlPassThruFields.setText( BaseMessages.getString( PKG, "TextFileInputDialog.PassThruFields.Label" ) ); + props.setLook( wlPassThruFields ); + fdlPassThruFields = new FormData(); + fdlPassThruFields.top = new FormAttachment( wAccFilenames, margin ); + fdlPassThruFields.left = new FormAttachment( 0, 0 ); + fdlPassThruFields.right = new FormAttachment( middle, -margin ); + wlPassThruFields.setLayoutData( fdlPassThruFields ); + wPassThruFields = new Button( gAccepting, SWT.CHECK ); + wPassThruFields.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.PassThruFields.Tooltip" ) ); + props.setLook( wPassThruFields ); + fdPassThruFields = new FormData(); + fdPassThruFields.top = new FormAttachment( wAccFilenames, margin ); + fdPassThruFields.left = new FormAttachment( middle, 0 ); + fdPassThruFields.right = new FormAttachment( 100, 0 ); + wPassThruFields.setLayoutData( fdPassThruFields ); + + // Which step to read from? + wlAccStep = new Label( gAccepting, SWT.RIGHT ); + wlAccStep.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptStep.Label" ) ); + props.setLook( wlAccStep ); + fdlAccStep = new FormData(); + fdlAccStep.top = new FormAttachment( wPassThruFields, margin ); + fdlAccStep.left = new FormAttachment( 0, 0 ); + fdlAccStep.right = new FormAttachment( middle, -margin ); + wlAccStep.setLayoutData( fdlAccStep ); + wAccStep = new CCombo( gAccepting, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wAccStep.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptStep.Tooltip" ) ); + props.setLook( wAccStep ); + fdAccStep = new FormData(); + fdAccStep.top = new FormAttachment( wPassThruFields, margin ); + fdAccStep.left = new FormAttachment( middle, 0 ); + fdAccStep.right = new FormAttachment( 100, 0 ); + wAccStep.setLayoutData( fdAccStep ); + + // Which field? + // + wlAccField = new Label( gAccepting, SWT.RIGHT ); + wlAccField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptField.Label" ) ); + props.setLook( wlAccField ); + fdlAccField = new FormData(); + fdlAccField.top = new FormAttachment( wAccStep, margin ); + fdlAccField.left = new FormAttachment( 0, 0 ); + fdlAccField.right = new FormAttachment( middle, -margin ); + wlAccField.setLayoutData( fdlAccField ); + wAccField = new Text( gAccepting, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wAccField.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.AcceptField.Tooltip" ) ); + props.setLook( wAccField ); + fdAccField = new FormData(); + fdAccField.top = new FormAttachment( wAccStep, margin ); + fdAccField.left = new FormAttachment( middle, 0 ); + fdAccField.right = new FormAttachment( 100, 0 ); + wAccField.setLayoutData( fdAccField ); + + // Fill in the source steps... + List prevSteps = transMeta.findPreviousSteps( transMeta.findStep( stepname ) ); + for ( StepMeta prevStep : prevSteps ) { + wAccStep.add( prevStep.getName() ); + } + + fdAccepting = new FormData(); + fdAccepting.left = new FormAttachment( 0, 0 ); + fdAccepting.right = new FormAttachment( 100, 0 ); + fdAccepting.bottom = new FormAttachment( wFirstHeader, -margin * 2 ); + gAccepting.setLayoutData( fdAccepting ); + + ColumnInfo[] colinfo = + new ColumnInfo[] { + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.FileDirColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.WildcardColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.Files.ExcludeWildcard.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.RequiredColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, YES_NO_COMBO ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.IncludeSubDirs.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, YES_NO_COMBO ) }; + + colinfo[0].setUsingVariables( true ); + colinfo[1].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.RegExpColumn.Column" ) ); + + // PDI-8664 + colinfo[1].setUsingVariables( true ); + + colinfo[2].setUsingVariables( true ); + colinfo[2].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.Files.ExcludeWildcard.Tooltip" ) ); + colinfo[3].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.RequiredColumn.Tooltip" ) ); + colinfo[4].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.IncludeSubDirs.Tooltip" ) ); + + wFilenameList = + new TableView( transMeta, wFileComp, SWT.FULL_SELECTION | SWT.SINGLE | SWT.BORDER, colinfo, 4, lsMod, props ); + props.setLook( wFilenameList ); + fdFilenameList = new FormData(); + fdFilenameList.left = new FormAttachment( middle, 0 ); + fdFilenameList.right = new FormAttachment( wbdFilename, -margin ); + fdFilenameList.top = new FormAttachment( wExcludeFilemask, margin ); + fdFilenameList.bottom = new FormAttachment( gAccepting, -margin ); + wFilenameList.setLayoutData( fdFilenameList ); + + fdFileComp = new FormData(); + fdFileComp.left = new FormAttachment( 0, 0 ); + fdFileComp.top = new FormAttachment( 0, 0 ); + fdFileComp.right = new FormAttachment( 100, 0 ); + fdFileComp.bottom = new FormAttachment( 100, 0 ); + wFileComp.setLayoutData( fdFileComp ); + + wFileComp.pack(); + Rectangle bounds = wFileComp.getBounds(); + + wFileSComp.setContent( wFileComp ); + wFileSComp.setExpandHorizontal( true ); + wFileSComp.setExpandVertical( true ); + wFileSComp.setMinWidth( bounds.width ); + wFileSComp.setMinHeight( bounds.height ); + + wFileTab.setControl( wFileSComp ); + + // /////////////////////////////////////////////////////////// + // / END OF FILE TAB + // /////////////////////////////////////////////////////////// + } + + private void addContentTab() { + // //////////////////////// + // START OF CONTENT TAB/// + // / + wContentTab = new CTabItem( wTabFolder, SWT.NONE ); + wContentTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ContentTab.TabTitle" ) ); + + FormLayout contentLayout = new FormLayout(); + contentLayout.marginWidth = 3; + contentLayout.marginHeight = 3; + + wContentSComp = new ScrolledComposite( wTabFolder, SWT.V_SCROLL | SWT.H_SCROLL ); + wContentSComp.setLayout( new FillLayout() ); + + wContentComp = new Composite( wContentSComp, SWT.NONE ); + props.setLook( wContentComp ); + wContentComp.setLayout( contentLayout ); + + // Filetype line + wlFiletype = new Label( wContentComp, SWT.RIGHT ); + wlFiletype.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Filetype.Label" ) ); + props.setLook( wlFiletype ); + fdlFiletype = new FormData(); + fdlFiletype.left = new FormAttachment( 0, 0 ); + fdlFiletype.top = new FormAttachment( 0, 0 ); + fdlFiletype.right = new FormAttachment( middle, -margin ); + wlFiletype.setLayoutData( fdlFiletype ); + wFiletype = new CCombo( wContentComp, SWT.BORDER | SWT.READ_ONLY ); + wFiletype.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Filetype.Label" ) ); + props.setLook( wFiletype ); + wFiletype.add( "CSV" ); + wFiletype.add( "Fixed" ); + wFiletype.select( 0 ); + wFiletype.addModifyListener( lsMod ); + fdFiletype = new FormData(); + fdFiletype.left = new FormAttachment( middle, 0 ); + fdFiletype.top = new FormAttachment( 0, 0 ); + fdFiletype.right = new FormAttachment( 100, 0 ); + wFiletype.setLayoutData( fdFiletype ); + + wlSeparator = new Label( wContentComp, SWT.RIGHT ); + wlSeparator.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Separator.Label" ) ); + props.setLook( wlSeparator ); + fdlSeparator = new FormData(); + fdlSeparator.left = new FormAttachment( 0, 0 ); + fdlSeparator.top = new FormAttachment( wFiletype, margin ); + fdlSeparator.right = new FormAttachment( middle, -margin ); + wlSeparator.setLayoutData( fdlSeparator ); + + wbSeparator = new Button( wContentComp, SWT.PUSH | SWT.CENTER ); + wbSeparator.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Delimiter.Button" ) ); + props.setLook( wbSeparator ); + fdbSeparator = new FormData(); + fdbSeparator.right = new FormAttachment( 100, 0 ); + fdbSeparator.top = new FormAttachment( wFiletype, 0 ); + wbSeparator.setLayoutData( fdbSeparator ); + wSeparator = new TextVar( transMeta, wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wSeparator ); + wSeparator.addModifyListener( lsMod ); + fdSeparator = new FormData(); + fdSeparator.top = new FormAttachment( wFiletype, margin ); + fdSeparator.left = new FormAttachment( middle, 0 ); + fdSeparator.right = new FormAttachment( wbSeparator, -margin ); + wSeparator.setLayoutData( fdSeparator ); + + // Enclosure + wlEnclosure = new Label( wContentComp, SWT.RIGHT ); + wlEnclosure.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Enclosure.Label" ) ); + props.setLook( wlEnclosure ); + fdlEnclosure = new FormData(); + fdlEnclosure.left = new FormAttachment( 0, 0 ); + fdlEnclosure.top = new FormAttachment( wSeparator, margin ); + fdlEnclosure.right = new FormAttachment( middle, -margin ); + wlEnclosure.setLayoutData( fdlEnclosure ); + wEnclosure = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wEnclosure ); + wEnclosure.addModifyListener( lsMod ); + fdEnclosure = new FormData(); + fdEnclosure.left = new FormAttachment( middle, 0 ); + fdEnclosure.top = new FormAttachment( wSeparator, margin ); + fdEnclosure.right = new FormAttachment( 100, 0 ); + wEnclosure.setLayoutData( fdEnclosure ); + + // Allow Enclosure breaks checkbox + wlEnclBreaks = new Label( wContentComp, SWT.RIGHT ); + wlEnclBreaks.setText( BaseMessages.getString( PKG, "TextFileInputDialog.EnclBreaks.Label" ) ); + props.setLook( wlEnclBreaks ); + fdlEnclBreaks = new FormData(); + fdlEnclBreaks.left = new FormAttachment( 0, 0 ); + fdlEnclBreaks.top = new FormAttachment( wEnclosure, margin ); + fdlEnclBreaks.right = new FormAttachment( middle, -margin ); + wlEnclBreaks.setLayoutData( fdlEnclBreaks ); + wEnclBreaks = new Button( wContentComp, SWT.CHECK ); + props.setLook( wEnclBreaks ); + fdEnclBreaks = new FormData(); + fdEnclBreaks.left = new FormAttachment( middle, 0 ); + fdEnclBreaks.top = new FormAttachment( wEnclosure, margin ); + wEnclBreaks.setLayoutData( fdEnclBreaks ); + + // Disable until the logic works... + wlEnclBreaks.setEnabled( false ); + wEnclBreaks.setEnabled( false ); + + // Escape + wlEscape = new Label( wContentComp, SWT.RIGHT ); + wlEscape.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Escape.Label" ) ); + props.setLook( wlEscape ); + fdlEscape = new FormData(); + fdlEscape.left = new FormAttachment( 0, 0 ); + fdlEscape.top = new FormAttachment( wEnclBreaks, margin ); + fdlEscape.right = new FormAttachment( middle, -margin ); + wlEscape.setLayoutData( fdlEscape ); + wEscape = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wEscape ); + wEscape.addModifyListener( lsMod ); + fdEscape = new FormData(); + fdEscape.left = new FormAttachment( middle, 0 ); + fdEscape.top = new FormAttachment( wEnclBreaks, margin ); + fdEscape.right = new FormAttachment( 100, 0 ); + wEscape.setLayoutData( fdEscape ); + + // Header checkbox + wlHeader = new Label( wContentComp, SWT.RIGHT ); + wlHeader.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Header.Label" ) ); + props.setLook( wlHeader ); + fdlHeader = new FormData(); + fdlHeader.left = new FormAttachment( 0, 0 ); + fdlHeader.top = new FormAttachment( wEscape, margin ); + fdlHeader.right = new FormAttachment( middle, -margin ); + wlHeader.setLayoutData( fdlHeader ); + wHeader = new Button( wContentComp, SWT.CHECK ); + props.setLook( wHeader ); + fdHeader = new FormData(); + fdHeader.left = new FormAttachment( middle, 0 ); + fdHeader.top = new FormAttachment( wEscape, margin ); + wHeader.setLayoutData( fdHeader ); + + // NrHeader + wlNrHeader = new Label( wContentComp, SWT.RIGHT ); + wlNrHeader.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NrHeader.Label" ) ); + props.setLook( wlNrHeader ); + fdlNrHeader = new FormData(); + fdlNrHeader.left = new FormAttachment( wHeader, margin ); + fdlNrHeader.top = new FormAttachment( wEscape, margin ); + wlNrHeader.setLayoutData( fdlNrHeader ); + wNrHeader = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wNrHeader.setTextLimit( 3 ); + props.setLook( wNrHeader ); + wNrHeader.addModifyListener( lsMod ); + fdNrHeader = new FormData(); + fdNrHeader.left = new FormAttachment( wlNrHeader, margin ); + fdNrHeader.top = new FormAttachment( wEscape, margin ); + fdNrHeader.right = new FormAttachment( 100, 0 ); + wNrHeader.setLayoutData( fdNrHeader ); + + wlFooter = new Label( wContentComp, SWT.RIGHT ); + wlFooter.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Footer.Label" ) ); + props.setLook( wlFooter ); + fdlFooter = new FormData(); + fdlFooter.left = new FormAttachment( 0, 0 ); + fdlFooter.top = new FormAttachment( wHeader, margin ); + fdlFooter.right = new FormAttachment( middle, -margin ); + wlFooter.setLayoutData( fdlFooter ); + wFooter = new Button( wContentComp, SWT.CHECK ); + props.setLook( wFooter ); + fdFooter = new FormData(); + fdFooter.left = new FormAttachment( middle, 0 ); + fdFooter.top = new FormAttachment( wHeader, margin ); + wFooter.setLayoutData( fdFooter ); + + // NrFooter + wlNrFooter = new Label( wContentComp, SWT.RIGHT ); + wlNrFooter.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NrFooter.Label" ) ); + props.setLook( wlNrFooter ); + fdlNrFooter = new FormData(); + fdlNrFooter.left = new FormAttachment( wFooter, margin ); + fdlNrFooter.top = new FormAttachment( wHeader, margin ); + wlNrFooter.setLayoutData( fdlNrFooter ); + wNrFooter = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wNrFooter.setTextLimit( 3 ); + props.setLook( wNrFooter ); + wNrFooter.addModifyListener( lsMod ); + fdNrFooter = new FormData(); + fdNrFooter.left = new FormAttachment( wlNrFooter, margin ); + fdNrFooter.top = new FormAttachment( wHeader, margin ); + fdNrFooter.right = new FormAttachment( 100, 0 ); + wNrFooter.setLayoutData( fdNrFooter ); + + // Wraps + wlWraps = new Label( wContentComp, SWT.RIGHT ); + wlWraps.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Wraps.Label" ) ); + props.setLook( wlWraps ); + fdlWraps = new FormData(); + fdlWraps.left = new FormAttachment( 0, 0 ); + fdlWraps.top = new FormAttachment( wFooter, margin ); + fdlWraps.right = new FormAttachment( middle, -margin ); + wlWraps.setLayoutData( fdlWraps ); + wWraps = new Button( wContentComp, SWT.CHECK ); + props.setLook( wWraps ); + fdWraps = new FormData(); + fdWraps.left = new FormAttachment( middle, 0 ); + fdWraps.top = new FormAttachment( wFooter, margin ); + wWraps.setLayoutData( fdWraps ); + + // NrWraps + wlNrWraps = new Label( wContentComp, SWT.RIGHT ); + wlNrWraps.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NrWraps.Label" ) ); + props.setLook( wlNrWraps ); + fdlNrWraps = new FormData(); + fdlNrWraps.left = new FormAttachment( wWraps, margin ); + fdlNrWraps.top = new FormAttachment( wFooter, margin ); + wlNrWraps.setLayoutData( fdlNrWraps ); + wNrWraps = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wNrWraps.setTextLimit( 3 ); + props.setLook( wNrWraps ); + wNrWraps.addModifyListener( lsMod ); + fdNrWraps = new FormData(); + fdNrWraps.left = new FormAttachment( wlNrWraps, margin ); + fdNrWraps.top = new FormAttachment( wFooter, margin ); + fdNrWraps.right = new FormAttachment( 100, 0 ); + wNrWraps.setLayoutData( fdNrWraps ); + + // Pages + wlLayoutPaged = new Label( wContentComp, SWT.RIGHT ); + wlLayoutPaged.setText( BaseMessages.getString( PKG, "TextFileInputDialog.LayoutPaged.Label" ) ); + props.setLook( wlLayoutPaged ); + fdlLayoutPaged = new FormData(); + fdlLayoutPaged.left = new FormAttachment( 0, 0 ); + fdlLayoutPaged.top = new FormAttachment( wWraps, margin ); + fdlLayoutPaged.right = new FormAttachment( middle, -margin ); + wlLayoutPaged.setLayoutData( fdlLayoutPaged ); + wLayoutPaged = new Button( wContentComp, SWT.CHECK ); + props.setLook( wLayoutPaged ); + fdLayoutPaged = new FormData(); + fdLayoutPaged.left = new FormAttachment( middle, 0 ); + fdLayoutPaged.top = new FormAttachment( wWraps, margin ); + wLayoutPaged.setLayoutData( fdLayoutPaged ); + + // Nr of lines per page + wlNrLinesPerPage = new Label( wContentComp, SWT.RIGHT ); + wlNrLinesPerPage.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NrLinesPerPage.Label" ) ); + props.setLook( wlNrLinesPerPage ); + fdlNrLinesPerPage = new FormData(); + fdlNrLinesPerPage.left = new FormAttachment( wLayoutPaged, margin ); + fdlNrLinesPerPage.top = new FormAttachment( wWraps, margin ); + wlNrLinesPerPage.setLayoutData( fdlNrLinesPerPage ); + wNrLinesPerPage = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wNrLinesPerPage.setTextLimit( 3 ); + props.setLook( wNrLinesPerPage ); + wNrLinesPerPage.addModifyListener( lsMod ); + fdNrLinesPerPage = new FormData(); + fdNrLinesPerPage.left = new FormAttachment( wlNrLinesPerPage, margin ); + fdNrLinesPerPage.top = new FormAttachment( wWraps, margin ); + fdNrLinesPerPage.right = new FormAttachment( 100, 0 ); + wNrLinesPerPage.setLayoutData( fdNrLinesPerPage ); + + // NrPages + wlNrLinesDocHeader = new Label( wContentComp, SWT.RIGHT ); + wlNrLinesDocHeader.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NrLinesDocHeader.Label" ) ); + props.setLook( wlNrLinesDocHeader ); + fdlNrLinesDocHeader = new FormData(); + fdlNrLinesDocHeader.left = new FormAttachment( wLayoutPaged, margin ); + fdlNrLinesDocHeader.top = new FormAttachment( wNrLinesPerPage, margin ); + wlNrLinesDocHeader.setLayoutData( fdlNrLinesDocHeader ); + wNrLinesDocHeader = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wNrLinesDocHeader.setTextLimit( 3 ); + props.setLook( wNrLinesDocHeader ); + wNrLinesDocHeader.addModifyListener( lsMod ); + fdNrLinesDocHeader = new FormData(); + + fdNrLinesDocHeader.left = new FormAttachment( wlNrLinesPerPage, margin ); + fdNrLinesDocHeader.top = new FormAttachment( wNrLinesPerPage, margin ); + fdNrLinesDocHeader.right = new FormAttachment( 100, 0 ); + wNrLinesDocHeader.setLayoutData( fdNrLinesDocHeader ); + + // Compression type (None, Zip or GZip + wlCompression = new Label( wContentComp, SWT.RIGHT ); + wlCompression.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Compression.Label" ) ); + props.setLook( wlCompression ); + fdlCompression = new FormData(); + fdlCompression.left = new FormAttachment( 0, 0 ); + fdlCompression.top = new FormAttachment( wNrLinesDocHeader, margin ); + fdlCompression.right = new FormAttachment( middle, -margin ); + wlCompression.setLayoutData( fdlCompression ); + wCompression = new CCombo( wContentComp, SWT.BORDER | SWT.READ_ONLY ); + wCompression.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Compression.Label" ) ); + wCompression.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.Compression.Tooltip" ) ); + props.setLook( wCompression ); + wCompression.setItems( CompressionProviderFactory.getInstance().getCompressionProviderNames() ); + + wCompression.addModifyListener( lsMod ); + fdCompression = new FormData(); + fdCompression.left = new FormAttachment( middle, 0 ); + fdCompression.top = new FormAttachment( wNrLinesDocHeader, margin ); + fdCompression.right = new FormAttachment( 100, 0 ); + wCompression.setLayoutData( fdCompression ); + + wlNoempty = new Label( wContentComp, SWT.RIGHT ); + wlNoempty.setText( BaseMessages.getString( PKG, "TextFileInputDialog.NoEmpty.Label" ) ); + props.setLook( wlNoempty ); + fdlNoempty = new FormData(); + fdlNoempty.left = new FormAttachment( 0, 0 ); + fdlNoempty.top = new FormAttachment( wCompression, margin ); + fdlNoempty.right = new FormAttachment( middle, -margin ); + wlNoempty.setLayoutData( fdlNoempty ); + wNoempty = new Button( wContentComp, SWT.CHECK ); + props.setLook( wNoempty ); + wNoempty.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.NoEmpty.Tooltip" ) ); + fdNoempty = new FormData(); + fdNoempty.left = new FormAttachment( middle, 0 ); + fdNoempty.top = new FormAttachment( wCompression, margin ); + fdNoempty.right = new FormAttachment( 100, 0 ); + wNoempty.setLayoutData( fdNoempty ); + + wlInclFilename = new Label( wContentComp, SWT.RIGHT ); + wlInclFilename.setText( BaseMessages.getString( PKG, "TextFileInputDialog.InclFilename.Label" ) ); + props.setLook( wlInclFilename ); + fdlInclFilename = new FormData(); + fdlInclFilename.left = new FormAttachment( 0, 0 ); + fdlInclFilename.top = new FormAttachment( wNoempty, margin ); + fdlInclFilename.right = new FormAttachment( middle, -margin ); + wlInclFilename.setLayoutData( fdlInclFilename ); + wInclFilename = new Button( wContentComp, SWT.CHECK ); + props.setLook( wInclFilename ); + wInclFilename.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.InclFilename.Tooltip" ) ); + fdInclFilename = new FormData(); + fdInclFilename.left = new FormAttachment( middle, 0 ); + fdInclFilename.top = new FormAttachment( wNoempty, margin ); + wInclFilename.setLayoutData( fdInclFilename ); + + wlInclFilenameField = new Label( wContentComp, SWT.LEFT ); + wlInclFilenameField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.InclFilenameField.Label" ) ); + props.setLook( wlInclFilenameField ); + fdlInclFilenameField = new FormData(); + fdlInclFilenameField.left = new FormAttachment( wInclFilename, margin ); + fdlInclFilenameField.top = new FormAttachment( wNoempty, margin ); + wlInclFilenameField.setLayoutData( fdlInclFilenameField ); + wInclFilenameField = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wInclFilenameField ); + wInclFilenameField.addModifyListener( lsMod ); + fdInclFilenameField = new FormData(); + fdInclFilenameField.left = new FormAttachment( wlInclFilenameField, margin ); + fdInclFilenameField.top = new FormAttachment( wNoempty, margin ); + fdInclFilenameField.right = new FormAttachment( 100, 0 ); + wInclFilenameField.setLayoutData( fdInclFilenameField ); + + wlInclRownum = new Label( wContentComp, SWT.RIGHT ); + wlInclRownum.setText( BaseMessages.getString( PKG, "TextFileInputDialog.InclRownum.Label" ) ); + props.setLook( wlInclRownum ); + fdlInclRownum = new FormData(); + fdlInclRownum.left = new FormAttachment( 0, 0 ); + fdlInclRownum.top = new FormAttachment( wInclFilenameField, margin ); + fdlInclRownum.right = new FormAttachment( middle, -margin ); + wlInclRownum.setLayoutData( fdlInclRownum ); + wInclRownum = new Button( wContentComp, SWT.CHECK ); + props.setLook( wInclRownum ); + wInclRownum.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.InclRownum.Tooltip" ) ); + fdRownum = new FormData(); + fdRownum.left = new FormAttachment( middle, 0 ); + fdRownum.top = new FormAttachment( wInclFilenameField, margin ); + wInclRownum.setLayoutData( fdRownum ); + + wlInclRownumField = new Label( wContentComp, SWT.RIGHT ); + wlInclRownumField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.InclRownumField.Label" ) ); + props.setLook( wlInclRownumField ); + fdlInclRownumField = new FormData(); + fdlInclRownumField.left = new FormAttachment( wInclRownum, margin ); + fdlInclRownumField.top = new FormAttachment( wInclFilenameField, margin ); + wlInclRownumField.setLayoutData( fdlInclRownumField ); + wInclRownumField = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wInclRownumField ); + wInclRownumField.addModifyListener( lsMod ); + fdInclRownumField = new FormData(); + fdInclRownumField.left = new FormAttachment( wlInclRownumField, margin ); + fdInclRownumField.top = new FormAttachment( wInclFilenameField, margin ); + fdInclRownumField.right = new FormAttachment( 100, 0 ); + wInclRownumField.setLayoutData( fdInclRownumField ); + + wlRownumByFileField = new Label( wContentComp, SWT.RIGHT ); + wlRownumByFileField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.RownumByFile.Label" ) ); + props.setLook( wlRownumByFileField ); + fdlRownumByFile = new FormData(); + fdlRownumByFile.left = new FormAttachment( wInclRownum, margin ); + fdlRownumByFile.top = new FormAttachment( wInclRownumField, margin ); + wlRownumByFileField.setLayoutData( fdlRownumByFile ); + wRownumByFile = new Button( wContentComp, SWT.CHECK ); + props.setLook( wRownumByFile ); + wRownumByFile.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.RownumByFile.Tooltip" ) ); + fdRownumByFile = new FormData(); + fdRownumByFile.left = new FormAttachment( wlRownumByFileField, margin ); + fdRownumByFile.top = new FormAttachment( wInclRownumField, margin ); + wRownumByFile.setLayoutData( fdRownumByFile ); + + wlFormat = new Label( wContentComp, SWT.RIGHT ); + wlFormat.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Format.Label" ) ); + props.setLook( wlFormat ); + fdlFormat = new FormData(); + fdlFormat.left = new FormAttachment( 0, 0 ); + fdlFormat.top = new FormAttachment( wRownumByFile, margin * 2 ); + fdlFormat.right = new FormAttachment( middle, -margin ); + wlFormat.setLayoutData( fdlFormat ); + wFormat = new CCombo( wContentComp, SWT.BORDER | SWT.READ_ONLY ); + wFormat.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Format.Label" ) ); + props.setLook( wFormat ); + wFormat.add( "DOS" ); + wFormat.add( "Unix" ); + wFormat.add( "mixed" ); + wFormat.select( 0 ); + wFormat.addModifyListener( lsMod ); + fdFormat = new FormData(); + fdFormat.left = new FormAttachment( middle, 0 ); + fdFormat.top = new FormAttachment( wRownumByFile, margin * 2 ); + fdFormat.right = new FormAttachment( 100, 0 ); + wFormat.setLayoutData( fdFormat ); + + wlEncoding = new Label( wContentComp, SWT.RIGHT ); + wlEncoding.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Encoding.Label" ) ); + props.setLook( wlEncoding ); + fdlEncoding = new FormData(); + fdlEncoding.left = new FormAttachment( 0, 0 ); + fdlEncoding.top = new FormAttachment( wFormat, margin ); + fdlEncoding.right = new FormAttachment( middle, -margin ); + wlEncoding.setLayoutData( fdlEncoding ); + wEncoding = new CCombo( wContentComp, SWT.BORDER | SWT.READ_ONLY ); + wEncoding.setEditable( true ); + props.setLook( wEncoding ); + wEncoding.addModifyListener( lsMod ); + fdEncoding = new FormData(); + fdEncoding.left = new FormAttachment( middle, 0 ); + fdEncoding.top = new FormAttachment( wFormat, margin ); + fdEncoding.right = new FormAttachment( 100, 0 ); + wEncoding.setLayoutData( fdEncoding ); + wEncoding.addFocusListener( new FocusListener() { + public void focusLost( org.eclipse.swt.events.FocusEvent e ) { + } + + public void focusGained( org.eclipse.swt.events.FocusEvent e ) { + Cursor busy = new Cursor( shell.getDisplay(), SWT.CURSOR_WAIT ); + shell.setCursor( busy ); + setEncodings(); + shell.setCursor( null ); + busy.dispose(); + } + } ); + + wlLimit = new Label( wContentComp, SWT.RIGHT ); + wlLimit.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Limit.Label" ) ); + props.setLook( wlLimit ); + fdlLimit = new FormData(); + fdlLimit.left = new FormAttachment( 0, 0 ); + fdlLimit.top = new FormAttachment( wEncoding, margin ); + fdlLimit.right = new FormAttachment( middle, -margin ); + wlLimit.setLayoutData( fdlLimit ); + wLimit = new Text( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wLimit ); + wLimit.addModifyListener( lsMod ); + fdLimit = new FormData(); + fdLimit.left = new FormAttachment( middle, 0 ); + fdLimit.top = new FormAttachment( wEncoding, margin ); + fdLimit.right = new FormAttachment( 100, 0 ); + wLimit.setLayoutData( fdLimit ); + + // Date Lenient checkbox + wlDateLenient = new Label( wContentComp, SWT.RIGHT ); + wlDateLenient.setText( BaseMessages.getString( PKG, "TextFileInputDialog.DateLenient.Label" ) ); + props.setLook( wlDateLenient ); + fdlDateLenient = new FormData(); + fdlDateLenient.left = new FormAttachment( 0, 0 ); + fdlDateLenient.top = new FormAttachment( wLimit, margin ); + fdlDateLenient.right = new FormAttachment( middle, -margin ); + wlDateLenient.setLayoutData( fdlDateLenient ); + wDateLenient = new Button( wContentComp, SWT.CHECK ); + wDateLenient.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.DateLenient.Tooltip" ) ); + props.setLook( wDateLenient ); + fdDateLenient = new FormData(); + fdDateLenient.left = new FormAttachment( middle, 0 ); + fdDateLenient.top = new FormAttachment( wLimit, margin ); + wDateLenient.setLayoutData( fdDateLenient ); + + wlDateLocale = new Label( wContentComp, SWT.RIGHT ); + wlDateLocale.setText( BaseMessages.getString( PKG, "TextFileInputDialog.DateLocale.Label" ) ); + props.setLook( wlDateLocale ); + fdlDateLocale = new FormData(); + fdlDateLocale.left = new FormAttachment( 0, 0 ); + fdlDateLocale.top = new FormAttachment( wDateLenient, margin ); + fdlDateLocale.right = new FormAttachment( middle, -margin ); + wlDateLocale.setLayoutData( fdlDateLocale ); + wDateLocale = new CCombo( wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + wDateLocale.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.DateLocale.Tooltip" ) ); + props.setLook( wDateLocale ); + wDateLocale.addModifyListener( lsMod ); + fdDateLocale = new FormData(); + fdDateLocale.left = new FormAttachment( middle, 0 ); + fdDateLocale.top = new FormAttachment( wDateLenient, margin ); + fdDateLocale.right = new FormAttachment( 100, 0 ); + wDateLocale.setLayoutData( fdDateLocale ); + wDateLocale.addFocusListener( new FocusListener() { + public void focusLost( org.eclipse.swt.events.FocusEvent e ) { + } + + public void focusGained( org.eclipse.swt.events.FocusEvent e ) { + Cursor busy = new Cursor( shell.getDisplay(), SWT.CURSOR_WAIT ); + shell.setCursor( busy ); + setLocales(); + shell.setCursor( null ); + busy.dispose(); + } + } ); + + // /////////////////////////////// + // START OF AddFileResult GROUP // + // /////////////////////////////// + + wAddFileResult = new Group( wContentComp, SWT.SHADOW_NONE ); + props.setLook( wAddFileResult ); + wAddFileResult.setText( BaseMessages.getString( PKG, "TextFileInputDialog.wAddFileResult.Label" ) ); + + FormLayout AddFileResultgroupLayout = new FormLayout(); + AddFileResultgroupLayout.marginWidth = 10; + AddFileResultgroupLayout.marginHeight = 10; + wAddFileResult.setLayout( AddFileResultgroupLayout ); + + wlAddResult = new Label( wAddFileResult, SWT.RIGHT ); + wlAddResult.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AddResult.Label" ) ); + props.setLook( wlAddResult ); + fdlAddResult = new FormData(); + fdlAddResult.left = new FormAttachment( 0, 0 ); + fdlAddResult.top = new FormAttachment( wDateLocale, margin ); + fdlAddResult.right = new FormAttachment( middle, -margin ); + wlAddResult.setLayoutData( fdlAddResult ); + wAddResult = new Button( wAddFileResult, SWT.CHECK ); + props.setLook( wAddResult ); + wAddResult.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.AddResult.Tooltip" ) ); + fdAddResult = new FormData(); + fdAddResult.left = new FormAttachment( middle, 0 ); + fdAddResult.top = new FormAttachment( wDateLocale, margin ); + wAddResult.setLayoutData( fdAddResult ); + + fdAddFileResult = new FormData(); + fdAddFileResult.left = new FormAttachment( 0, margin ); + fdAddFileResult.top = new FormAttachment( wDateLocale, margin ); + fdAddFileResult.right = new FormAttachment( 100, -margin ); + wAddFileResult.setLayoutData( fdAddFileResult ); + + // /////////////////////////////////////////////////////////// + // / END OF AddFileResult GROUP + // /////////////////////////////////////////////////////////// + + wContentComp.pack(); + // What's the size: + Rectangle bounds = wContentComp.getBounds(); + + wContentSComp.setContent( wContentComp ); + wContentSComp.setExpandHorizontal( true ); + wContentSComp.setExpandVertical( true ); + wContentSComp.setMinWidth( bounds.width ); + wContentSComp.setMinHeight( bounds.height ); + + fdContentComp = new FormData(); + fdContentComp.left = new FormAttachment( 0, 0 ); + fdContentComp.top = new FormAttachment( 0, 0 ); + fdContentComp.right = new FormAttachment( 100, 0 ); + fdContentComp.bottom = new FormAttachment( 100, 0 ); + wContentComp.setLayoutData( fdContentComp ); + + wContentTab.setControl( wContentSComp ); + + // /////////////////////////////////////////////////////////// + // / END OF CONTENT TAB + // /////////////////////////////////////////////////////////// + + } + + protected void setLocales() { + Locale[] locale = Locale.getAvailableLocales(); + dateLocale = new String[locale.length]; + for ( int i = 0; i < locale.length; i++ ) { + dateLocale[i] = locale[i].toString(); + } + if ( dateLocale != null ) { + wDateLocale.setItems( dateLocale ); + } + } + + private void addErrorTab() { + // //////////////////////// + // START OF ERROR TAB /// + // / + wErrorTab = new CTabItem( wTabFolder, SWT.NONE ); + wErrorTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorTab.TabTitle" ) ); + + wErrorSComp = new ScrolledComposite( wTabFolder, SWT.V_SCROLL | SWT.H_SCROLL ); + wErrorSComp.setLayout( new FillLayout() ); + + FormLayout errorLayout = new FormLayout(); + errorLayout.marginWidth = 3; + errorLayout.marginHeight = 3; + + wErrorComp = new Composite( wErrorSComp, SWT.NONE ); + props.setLook( wErrorComp ); + wErrorComp.setLayout( errorLayout ); + + // ERROR HANDLING... + // ErrorIgnored? + wlErrorIgnored = new Label( wErrorComp, SWT.RIGHT ); + wlErrorIgnored.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorIgnored.Label" ) ); + props.setLook( wlErrorIgnored ); + fdlErrorIgnored = new FormData(); + fdlErrorIgnored.left = new FormAttachment( 0, 0 ); + fdlErrorIgnored.top = new FormAttachment( 0, margin ); + fdlErrorIgnored.right = new FormAttachment( middle, -margin ); + wlErrorIgnored.setLayoutData( fdlErrorIgnored ); + wErrorIgnored = new Button( wErrorComp, SWT.CHECK ); + props.setLook( wErrorIgnored ); + wErrorIgnored.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorIgnored.Tooltip" ) ); + fdErrorIgnored = new FormData(); + fdErrorIgnored.left = new FormAttachment( middle, 0 ); + fdErrorIgnored.top = new FormAttachment( 0, margin ); + wErrorIgnored.setLayoutData( fdErrorIgnored ); + + // Skip bad files? + wlSkipBadFiles = new Label( wErrorComp, SWT.RIGHT ); + wlSkipBadFiles.setText( BaseMessages.getString( PKG, "TextFileInputDialog.SkipBadFiles.Label" ) ); + props.setLook( wlSkipBadFiles ); + fdlSkipBadFiles = new FormData(); + fdlSkipBadFiles.left = new FormAttachment( 0, 0 ); + fdlSkipBadFiles.top = new FormAttachment( wErrorIgnored, margin ); + fdlSkipBadFiles.right = new FormAttachment( middle, -margin ); + wlSkipBadFiles.setLayoutData( fdlSkipBadFiles ); + wSkipBadFiles = new Button( wErrorComp, SWT.CHECK ); + props.setLook( wSkipBadFiles ); + wSkipBadFiles.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.SkipBadFiles.Tooltip" ) ); + fdSkipBadFiles = new FormData(); + fdSkipBadFiles.left = new FormAttachment( middle, 0 ); + fdSkipBadFiles.top = new FormAttachment( wErrorIgnored, margin ); + wSkipBadFiles.setLayoutData( fdSkipBadFiles ); + + // field for rejected file + wlBadFileField = new Label( wErrorComp, SWT.RIGHT ); + wlBadFileField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.BadFileField.Label" ) ); + props.setLook( wlBadFileField ); + fdlBadFileField = new FormData(); + fdlBadFileField.left = new FormAttachment( 0, 0 ); + fdlBadFileField.top = new FormAttachment( wSkipBadFiles, margin ); + fdlBadFileField.right = new FormAttachment( middle, -margin ); + wlBadFileField.setLayoutData( fdlBadFileField ); + wBadFileField = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wBadFileField ); + wBadFileField.addModifyListener( lsMod ); + fdBadFileField = new FormData(); + fdBadFileField.left = new FormAttachment( middle, 0 ); + fdBadFileField.top = new FormAttachment( wSkipBadFiles, margin ); + fdBadFileField.right = new FormAttachment( 100, 0 ); + wBadFileField.setLayoutData( fdBadFileField ); + + // field for file error messsage + wlBadFileMessageField = new Label( wErrorComp, SWT.RIGHT ); + wlBadFileMessageField.setText( BaseMessages.getString( PKG, "TextFileInputDialog.BadFileMessageField.Label" ) ); + props.setLook( wlBadFileMessageField ); + fdlBadFileMessageField = new FormData(); + fdlBadFileMessageField.left = new FormAttachment( 0, 0 ); + fdlBadFileMessageField.top = new FormAttachment( wBadFileField, margin ); + fdlBadFileMessageField.right = new FormAttachment( middle, -margin ); + wlBadFileMessageField.setLayoutData( fdlBadFileMessageField ); + wBadFileMessageField = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wBadFileMessageField ); + wBadFileMessageField.addModifyListener( lsMod ); + fdBadFileMessageField = new FormData(); + fdBadFileMessageField.left = new FormAttachment( middle, 0 ); + fdBadFileMessageField.top = new FormAttachment( wBadFileField, margin ); + fdBadFileMessageField.right = new FormAttachment( 100, 0 ); + wBadFileMessageField.setLayoutData( fdBadFileMessageField ); + + // Skip error lines? + wlSkipErrorLines = new Label( wErrorComp, SWT.RIGHT ); + wlSkipErrorLines.setText( BaseMessages.getString( PKG, "TextFileInputDialog.SkipErrorLines.Label" ) ); + props.setLook( wlSkipErrorLines ); + fdlSkipErrorLines = new FormData(); + fdlSkipErrorLines.left = new FormAttachment( 0, 0 ); + fdlSkipErrorLines.top = new FormAttachment( wBadFileMessageField, margin ); + fdlSkipErrorLines.right = new FormAttachment( middle, -margin ); + wlSkipErrorLines.setLayoutData( fdlSkipErrorLines ); + wSkipErrorLines = new Button( wErrorComp, SWT.CHECK ); + props.setLook( wSkipErrorLines ); + wSkipErrorLines.setToolTipText( BaseMessages.getString( PKG, "TextFileInputDialog.SkipErrorLines.Tooltip" ) ); + fdSkipErrorLines = new FormData(); + fdSkipErrorLines.left = new FormAttachment( middle, 0 ); + fdSkipErrorLines.top = new FormAttachment( wBadFileMessageField, margin ); + wSkipErrorLines.setLayoutData( fdSkipErrorLines ); + + wlErrorCount = new Label( wErrorComp, SWT.RIGHT ); + wlErrorCount.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorCount.Label" ) ); + props.setLook( wlErrorCount ); + fdlErrorCount = new FormData(); + fdlErrorCount.left = new FormAttachment( 0, 0 ); + fdlErrorCount.top = new FormAttachment( wSkipErrorLines, margin ); + fdlErrorCount.right = new FormAttachment( middle, -margin ); + wlErrorCount.setLayoutData( fdlErrorCount ); + wErrorCount = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wErrorCount ); + wErrorCount.addModifyListener( lsMod ); + fdErrorCount = new FormData(); + fdErrorCount.left = new FormAttachment( middle, 0 ); + fdErrorCount.top = new FormAttachment( wSkipErrorLines, margin ); + fdErrorCount.right = new FormAttachment( 100, 0 ); + wErrorCount.setLayoutData( fdErrorCount ); + + wlErrorFields = new Label( wErrorComp, SWT.RIGHT ); + wlErrorFields.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorFields.Label" ) ); + props.setLook( wlErrorFields ); + fdlErrorFields = new FormData(); + fdlErrorFields.left = new FormAttachment( 0, 0 ); + fdlErrorFields.top = new FormAttachment( wErrorCount, margin ); + fdlErrorFields.right = new FormAttachment( middle, -margin ); + wlErrorFields.setLayoutData( fdlErrorFields ); + wErrorFields = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wErrorFields ); + wErrorFields.addModifyListener( lsMod ); + fdErrorFields = new FormData(); + fdErrorFields.left = new FormAttachment( middle, 0 ); + fdErrorFields.top = new FormAttachment( wErrorCount, margin ); + fdErrorFields.right = new FormAttachment( 100, 0 ); + wErrorFields.setLayoutData( fdErrorFields ); + + wlErrorText = new Label( wErrorComp, SWT.RIGHT ); + wlErrorText.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorText.Label" ) ); + props.setLook( wlErrorText ); + fdlErrorText = new FormData(); + fdlErrorText.left = new FormAttachment( 0, 0 ); + fdlErrorText.top = new FormAttachment( wErrorFields, margin ); + fdlErrorText.right = new FormAttachment( middle, -margin ); + wlErrorText.setLayoutData( fdlErrorText ); + wErrorText = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wErrorText ); + wErrorText.addModifyListener( lsMod ); + fdErrorText = new FormData(); + fdErrorText.left = new FormAttachment( middle, 0 ); + fdErrorText.top = new FormAttachment( wErrorFields, margin ); + fdErrorText.right = new FormAttachment( 100, 0 ); + wErrorText.setLayoutData( fdErrorText ); + + // Bad lines files directory + extension + Control previous = wErrorText; + + // BadDestDir line + wlWarnDestDir = new Label( wErrorComp, SWT.RIGHT ); + wlWarnDestDir.setText( BaseMessages.getString( PKG, "TextFileInputDialog.WarnDestDir.Label" ) ); + props.setLook( wlWarnDestDir ); + fdlWarnDestDir = new FormData(); + fdlWarnDestDir.left = new FormAttachment( 0, 0 ); + fdlWarnDestDir.top = new FormAttachment( previous, margin * 4 ); + fdlWarnDestDir.right = new FormAttachment( middle, -margin ); + wlWarnDestDir.setLayoutData( fdlWarnDestDir ); + + wbbWarnDestDir = new Button( wErrorComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbbWarnDestDir ); + wbbWarnDestDir.setText( BaseMessages.getString( PKG, "System.Button.Browse" ) ); + wbbWarnDestDir.setToolTipText( BaseMessages.getString( PKG, "System.Tooltip.BrowseForDir" ) ); + fdbBadDestDir = new FormData(); + fdbBadDestDir.right = new FormAttachment( 100, 0 ); + fdbBadDestDir.top = new FormAttachment( previous, margin * 4 ); + wbbWarnDestDir.setLayoutData( fdbBadDestDir ); + + wWarnExt = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wWarnExt ); + wWarnExt.addModifyListener( lsMod ); + fdWarnDestExt = new FormData(); + fdWarnDestExt.left = new FormAttachment( wbbWarnDestDir, -150 ); + fdWarnDestExt.right = new FormAttachment( wbbWarnDestDir, -margin ); + fdWarnDestExt.top = new FormAttachment( previous, margin * 4 ); + wWarnExt.setLayoutData( fdWarnDestExt ); + + wlWarnExt = new Label( wErrorComp, SWT.RIGHT ); + wlWarnExt.setText( BaseMessages.getString( PKG, "System.Label.Extension" ) ); + props.setLook( wlWarnExt ); + fdlWarnDestExt = new FormData(); + fdlWarnDestExt.top = new FormAttachment( previous, margin * 4 ); + fdlWarnDestExt.right = new FormAttachment( wWarnExt, -margin ); + wlWarnExt.setLayoutData( fdlWarnDestExt ); + + wWarnDestDir = new TextVar( transMeta, wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wWarnDestDir ); + wWarnDestDir.addModifyListener( lsMod ); + fdBadDestDir = new FormData(); + fdBadDestDir.left = new FormAttachment( middle, 0 ); + fdBadDestDir.right = new FormAttachment( wlWarnExt, -margin ); + fdBadDestDir.top = new FormAttachment( previous, margin * 4 ); + wWarnDestDir.setLayoutData( fdBadDestDir ); + + // Listen to the Browse... button + wbbWarnDestDir + .addSelectionListener( + OldDirectoryDialogButtonListenerFactory.getSelectionAdapter( shell, wWarnDestDir.getTextWidget() ) ); + + // Whenever something changes, set the tooltip to the expanded version of the directory: + wWarnDestDir.addModifyListener( getModifyListenerTooltipText( wWarnDestDir.getTextWidget() ) ); + + // Error lines files directory + extension + previous = wWarnDestDir; + + // ErrorDestDir line + wlErrorDestDir = new Label( wErrorComp, SWT.RIGHT ); + wlErrorDestDir.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ErrorDestDir.Label" ) ); + props.setLook( wlErrorDestDir ); + fdlErrorDestDir = new FormData(); + fdlErrorDestDir.left = new FormAttachment( 0, 0 ); + fdlErrorDestDir.top = new FormAttachment( previous, margin ); + fdlErrorDestDir.right = new FormAttachment( middle, -margin ); + wlErrorDestDir.setLayoutData( fdlErrorDestDir ); + + wbbErrorDestDir = new Button( wErrorComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbbErrorDestDir ); + wbbErrorDestDir.setText( BaseMessages.getString( PKG, "System.Button.Browse" ) ); + wbbErrorDestDir.setToolTipText( BaseMessages.getString( PKG, "System.Tooltip.BrowseForDir" ) ); + fdbErrorDestDir = new FormData(); + fdbErrorDestDir.right = new FormAttachment( 100, 0 ); + fdbErrorDestDir.top = new FormAttachment( previous, margin ); + wbbErrorDestDir.setLayoutData( fdbErrorDestDir ); + + wErrorExt = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wErrorExt ); + wErrorExt.addModifyListener( lsMod ); + fdErrorDestExt = new FormData(); + fdErrorDestExt.left = new FormAttachment( wWarnExt, 0, SWT.LEFT ); + fdErrorDestExt.right = new FormAttachment( wWarnExt, 0, SWT.RIGHT ); + fdErrorDestExt.top = new FormAttachment( previous, margin ); + wErrorExt.setLayoutData( fdErrorDestExt ); + + wlErrorExt = new Label( wErrorComp, SWT.RIGHT ); + wlErrorExt.setText( BaseMessages.getString( PKG, "System.Label.Extension" ) ); + props.setLook( wlErrorExt ); + fdlErrorDestExt = new FormData(); + fdlErrorDestExt.top = new FormAttachment( previous, margin ); + fdlErrorDestExt.right = new FormAttachment( wErrorExt, -margin ); + wlErrorExt.setLayoutData( fdlErrorDestExt ); + + wErrorDestDir = new TextVar( transMeta, wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wErrorDestDir ); + wErrorDestDir.addModifyListener( lsMod ); + fdErrorDestDir = new FormData(); + fdErrorDestDir.left = new FormAttachment( middle, 0 ); + fdErrorDestDir.right = new FormAttachment( wlErrorExt, -margin ); + fdErrorDestDir.top = new FormAttachment( previous, margin ); + wErrorDestDir.setLayoutData( fdErrorDestDir ); + + // Listen to the Browse... button + wbbErrorDestDir.addSelectionListener( OldDirectoryDialogButtonListenerFactory.getSelectionAdapter( shell, + wErrorDestDir.getTextWidget() ) ); + + // Whenever something changes, set the tooltip to the expanded version of the directory: + wErrorDestDir.addModifyListener( getModifyListenerTooltipText( wErrorDestDir.getTextWidget() ) ); + + // Data Error lines files directory + extension + previous = wErrorDestDir; + + // LineNrDestDir line + wlLineNrDestDir = new Label( wErrorComp, SWT.RIGHT ); + wlLineNrDestDir.setText( BaseMessages.getString( PKG, "TextFileInputDialog.LineNrDestDir.Label" ) ); + props.setLook( wlLineNrDestDir ); + fdlLineNrDestDir = new FormData(); + fdlLineNrDestDir.left = new FormAttachment( 0, 0 ); + fdlLineNrDestDir.top = new FormAttachment( previous, margin ); + fdlLineNrDestDir.right = new FormAttachment( middle, -margin ); + wlLineNrDestDir.setLayoutData( fdlLineNrDestDir ); + + wbbLineNrDestDir = new Button( wErrorComp, SWT.PUSH | SWT.CENTER ); + props.setLook( wbbLineNrDestDir ); + wbbLineNrDestDir.setText( BaseMessages.getString( PKG, "System.Button.Browse" ) ); + wbbLineNrDestDir.setToolTipText( BaseMessages.getString( PKG, "System.Tooltip.Browse" ) ); + fdbLineNrDestDir = new FormData(); + fdbLineNrDestDir.right = new FormAttachment( 100, 0 ); + fdbLineNrDestDir.top = new FormAttachment( previous, margin ); + wbbLineNrDestDir.setLayoutData( fdbLineNrDestDir ); + + wLineNrExt = new Text( wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wLineNrExt ); + wLineNrExt.addModifyListener( lsMod ); + fdLineNrDestExt = new FormData(); + fdLineNrDestExt.left = new FormAttachment( wErrorExt, 0, SWT.LEFT ); + fdLineNrDestExt.right = new FormAttachment( wErrorExt, 0, SWT.RIGHT ); + fdLineNrDestExt.top = new FormAttachment( previous, margin ); + wLineNrExt.setLayoutData( fdLineNrDestExt ); + + wlLineNrExt = new Label( wErrorComp, SWT.RIGHT ); + wlLineNrExt.setText( BaseMessages.getString( PKG, "System.Label.Extension" ) ); + props.setLook( wlLineNrExt ); + fdlLineNrDestExt = new FormData(); + fdlLineNrDestExt.top = new FormAttachment( previous, margin ); + fdlLineNrDestExt.right = new FormAttachment( wLineNrExt, -margin ); + wlLineNrExt.setLayoutData( fdlLineNrDestExt ); + + wLineNrDestDir = new TextVar( transMeta, wErrorComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wLineNrDestDir ); + wLineNrDestDir.addModifyListener( lsMod ); + fdLineNrDestDir = new FormData(); + fdLineNrDestDir.left = new FormAttachment( middle, 0 ); + fdLineNrDestDir.right = new FormAttachment( wlLineNrExt, -margin ); + fdLineNrDestDir.top = new FormAttachment( previous, margin ); + wLineNrDestDir.setLayoutData( fdLineNrDestDir ); + + // Listen to the Browse... button + wbbLineNrDestDir.addSelectionListener( OldDirectoryDialogButtonListenerFactory.getSelectionAdapter( shell, + wLineNrDestDir.getTextWidget() ) ); + + // Whenever something changes, set the tooltip to the expanded version of the directory: + wLineNrDestDir.addModifyListener( getModifyListenerTooltipText( wLineNrDestDir.getTextWidget() ) ); + + fdErrorComp = new FormData(); + fdErrorComp.left = new FormAttachment( 0, 0 ); + fdErrorComp.top = new FormAttachment( 0, 0 ); + fdErrorComp.right = new FormAttachment( 100, 0 ); + fdErrorComp.bottom = new FormAttachment( 100, 0 ); + wErrorComp.setLayoutData( fdErrorComp ); + + wErrorComp.pack(); + // What's the size: + Rectangle bounds = wErrorComp.getBounds(); + + wErrorSComp.setContent( wErrorComp ); + wErrorSComp.setExpandHorizontal( true ); + wErrorSComp.setExpandVertical( true ); + wErrorSComp.setMinWidth( bounds.width ); + wErrorSComp.setMinHeight( bounds.height ); + + wErrorTab.setControl( wErrorSComp ); + + // /////////////////////////////////////////////////////////// + // / END OF CONTENT TAB + // /////////////////////////////////////////////////////////// + + } + + private void addFiltersTabs() { + // Filters tab... + // + wFilterTab = new CTabItem( wTabFolder, SWT.NONE ); + wFilterTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FilterTab.TabTitle" ) ); + + FormLayout FilterLayout = new FormLayout(); + FilterLayout.marginWidth = Const.FORM_MARGIN; + FilterLayout.marginHeight = Const.FORM_MARGIN; + + wFilterComp = new Composite( wTabFolder, SWT.NONE ); + wFilterComp.setLayout( FilterLayout ); + props.setLook( wFilterComp ); + + final int FilterRows = input.getFilter().length; + + ColumnInfo[] colinf = + new ColumnInfo[] { + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.FilterStringColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.FilterPositionColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.StopOnFilterColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, YES_NO_COMBO ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.FilterPositiveColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, YES_NO_COMBO ) }; + + colinf[2].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.StopOnFilterColumn.Tooltip" ) ); + colinf[3].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.FilterPositiveColumn.Tooltip" ) ); + + wFilter = new TableView( transMeta, wFilterComp, SWT.FULL_SELECTION | SWT.MULTI, colinf, FilterRows, lsMod, props ); + + fdFilter = new FormData(); + fdFilter.left = new FormAttachment( 0, 0 ); + fdFilter.top = new FormAttachment( 0, 0 ); + fdFilter.right = new FormAttachment( 100, 0 ); + fdFilter.bottom = new FormAttachment( 100, 0 ); + wFilter.setLayoutData( fdFilter ); + + fdFilterComp = new FormData(); + fdFilterComp.left = new FormAttachment( 0, 0 ); + fdFilterComp.top = new FormAttachment( 0, 0 ); + fdFilterComp.right = new FormAttachment( 100, 0 ); + fdFilterComp.bottom = new FormAttachment( 100, 0 ); + wFilterComp.setLayoutData( fdFilterComp ); + + wFilterComp.layout(); + wFilterTab.setControl( wFilterComp ); + } + + private void addFieldsTabs() { + // Fields tab... + // + wFieldsTab = new CTabItem( wTabFolder, SWT.NONE ); + wFieldsTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.FieldsTab.TabTitle" ) ); + + FormLayout fieldsLayout = new FormLayout(); + fieldsLayout.marginWidth = Const.FORM_MARGIN; + fieldsLayout.marginHeight = Const.FORM_MARGIN; + + wFieldsComp = new Composite( wTabFolder, SWT.NONE ); + wFieldsComp.setLayout( fieldsLayout ); + props.setLook( wFieldsComp ); + + wGet = new Button( wFieldsComp, SWT.PUSH ); + wGet.setText( BaseMessages.getString( PKG, "System.Button.GetFields" ) ); + fdGet = new FormData(); + fdGet.left = new FormAttachment( 50, 0 ); + fdGet.bottom = new FormAttachment( 100, 0 ); + wGet.setLayoutData( fdGet ); + + final int FieldsRows = input.getInputFields().length; + + ColumnInfo[] colinf = + new ColumnInfo[] { + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.NameColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.TypeColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMeta.getTypes(), true ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.FormatColumn.Column" ), + ColumnInfo.COLUMN_TYPE_FORMAT, 2 ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.PositionColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.LengthColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.PrecisionColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.CurrencyColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.DecimalColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.GroupColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.NullIfColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.IfNullColumn.Column" ), + ColumnInfo.COLUMN_TYPE_TEXT, false ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.TrimTypeColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMeta.trimTypeDesc, true ), + new ColumnInfo( BaseMessages.getString( PKG, "TextFileInputDialog.RepeatColumn.Column" ), + ColumnInfo.COLUMN_TYPE_CCOMBO, new String[] { BaseMessages.getString( PKG, "System.Combo.Yes" ), + BaseMessages.getString( PKG, "System.Combo.No" ) }, true ) }; + + colinf[12].setToolTip( BaseMessages.getString( PKG, "TextFileInputDialog.RepeatColumn.Tooltip" ) ); + + wFields = new TableView( transMeta, wFieldsComp, SWT.FULL_SELECTION | SWT.MULTI, colinf, FieldsRows, lsMod, props ); + + fdFields = new FormData(); + fdFields.left = new FormAttachment( 0, 0 ); + fdFields.top = new FormAttachment( 0, 0 ); + fdFields.right = new FormAttachment( 100, 0 ); + fdFields.bottom = new FormAttachment( wGet, -margin ); + wFields.setLayoutData( fdFields ); + + fdFieldsComp = new FormData(); + fdFieldsComp.left = new FormAttachment( 0, 0 ); + fdFieldsComp.top = new FormAttachment( 0, 0 ); + fdFieldsComp.right = new FormAttachment( 100, 0 ); + fdFieldsComp.bottom = new FormAttachment( 100, 0 ); + wFieldsComp.setLayoutData( fdFieldsComp ); + + wFieldsComp.layout(); + wFieldsTab.setControl( wFieldsComp ); + } + + public void setFlags() { + boolean accept = wAccFilenames.getSelection(); + wlPassThruFields.setEnabled( accept ); + wPassThruFields.setEnabled( accept ); + if ( !wAccFilenames.getSelection() ) { + wPassThruFields.setSelection( false ); + } + wlAccField.setEnabled( accept ); + wAccField.setEnabled( accept ); + wlAccStep.setEnabled( accept ); + wAccStep.setEnabled( accept ); + + wlFilename.setEnabled( !accept ); + wbbFilename.setEnabled( !accept ); // Browse: add file or directory + wbdFilename.setEnabled( !accept ); // Delete + wbeFilename.setEnabled( !accept ); // Edit + wbaFilename.setEnabled( !accept ); // Add or change + wFilename.setEnabled( !accept ); + wlFilenameList.setEnabled( !accept ); + wFilenameList.setEnabled( !accept ); + wlFilemask.setEnabled( !accept ); + wFilemask.setEnabled( !accept ); + wbShowFiles.setEnabled( !accept ); + + // Keep this one active: use the sample in the file list + // wPreview.setEnabled(!accept); + + wFirst.setEnabled( !accept ); + wFirstHeader.setEnabled( !accept ); + + wlInclFilenameField.setEnabled( wInclFilename.getSelection() ); + wInclFilenameField.setEnabled( wInclFilename.getSelection() ); + + wlInclRownumField.setEnabled( wInclRownum.getSelection() ); + wInclRownumField.setEnabled( wInclRownum.getSelection() ); + wlRownumByFileField.setEnabled( wInclRownum.getSelection() ); + wRownumByFile.setEnabled( wInclRownum.getSelection() ); + + // Error handling tab... + wlSkipErrorLines.setEnabled( wErrorIgnored.getSelection() ); + wSkipBadFiles.setEnabled( wErrorIgnored.getSelection() ); + wBadFileField.setEnabled( wErrorIgnored.getSelection() && wSkipBadFiles.getSelection() ); + wBadFileMessageField.setEnabled( wErrorIgnored.getSelection() && wSkipBadFiles.getSelection() ); + wSkipErrorLines.setEnabled( wErrorIgnored.getSelection() ); + wlErrorCount.setEnabled( wErrorIgnored.getSelection() ); + wErrorCount.setEnabled( wErrorIgnored.getSelection() ); + wlErrorFields.setEnabled( wErrorIgnored.getSelection() ); + wErrorFields.setEnabled( wErrorIgnored.getSelection() ); + wlErrorText.setEnabled( wErrorIgnored.getSelection() ); + wErrorText.setEnabled( wErrorIgnored.getSelection() ); + + wlWarnDestDir.setEnabled( wErrorIgnored.getSelection() ); + wWarnDestDir.setEnabled( wErrorIgnored.getSelection() ); + wlWarnExt.setEnabled( wErrorIgnored.getSelection() ); + wWarnExt.setEnabled( wErrorIgnored.getSelection() ); + wbbWarnDestDir.setEnabled( wErrorIgnored.getSelection() ); + + wlErrorDestDir.setEnabled( wErrorIgnored.getSelection() ); + wErrorDestDir.setEnabled( wErrorIgnored.getSelection() ); + wlErrorExt.setEnabled( wErrorIgnored.getSelection() ); + wErrorExt.setEnabled( wErrorIgnored.getSelection() ); + wbbErrorDestDir.setEnabled( wErrorIgnored.getSelection() ); + + wlLineNrDestDir.setEnabled( wErrorIgnored.getSelection() ); + wLineNrDestDir.setEnabled( wErrorIgnored.getSelection() ); + wlLineNrExt.setEnabled( wErrorIgnored.getSelection() ); + wLineNrExt.setEnabled( wErrorIgnored.getSelection() ); + wbbLineNrDestDir.setEnabled( wErrorIgnored.getSelection() ); + + wlNrHeader.setEnabled( wHeader.getSelection() ); + wNrHeader.setEnabled( wHeader.getSelection() ); + wlNrFooter.setEnabled( wFooter.getSelection() ); + wNrFooter.setEnabled( wFooter.getSelection() ); + wlNrWraps.setEnabled( wWraps.getSelection() ); + wNrWraps.setEnabled( wWraps.getSelection() ); + + wlNrLinesPerPage.setEnabled( wLayoutPaged.getSelection() ); + wNrLinesPerPage.setEnabled( wLayoutPaged.getSelection() ); + wlNrLinesDocHeader.setEnabled( wLayoutPaged.getSelection() ); + wNrLinesDocHeader.setEnabled( wLayoutPaged.getSelection() ); + } + + /** + * Read the data from the OldTextFileInputMeta object and show it in this dialog. + * + * @param meta + * The OldTextFileInputMeta object to obtain the data from. + */ + public void getData( OldTextFileInputMeta meta ) { + wAccFilenames.setSelection( meta.isAcceptingFilenames() ); + wPassThruFields.setSelection( meta.isPassingThruFields() ); + if ( meta.getAcceptingField() != null ) { + wAccField.setText( meta.getAcceptingField() ); + } + if ( meta.getAcceptingStep() != null ) { + wAccStep.setText( meta.getAcceptingStep().getName() ); + } + + if ( meta.getFileName() != null ) { + wFilenameList.removeAll(); + + for ( int i = 0; i < meta.getFileName().length; i++ ) { + wFilenameList + .add( meta.getFileName()[ i ], meta.getFileMask()[ i ], meta.getExludeFileMask()[ i ], + meta.getRequiredFilesDesc( meta.getFileRequired()[ i ] ), + meta.getRequiredFilesDesc( meta.getIncludeSubFolders()[ i ] ) ); + } + wFilenameList.removeEmptyRows(); + wFilenameList.setRowNums(); + wFilenameList.optWidth( true ); + } + if ( meta.getFileType() != null ) { + wFiletype.setText( meta.getFileType() ); + } + if ( meta.getSeparator() != null ) { + wSeparator.setText( meta.getSeparator() ); + } + if ( meta.getEnclosure() != null ) { + wEnclosure.setText( meta.getEnclosure() ); + } + if ( meta.getEscapeCharacter() != null ) { + wEscape.setText( meta.getEscapeCharacter() ); + } + wHeader.setSelection( meta.hasHeader() ); + wNrHeader.setText( "" + meta.getNrHeaderLines() ); + wFooter.setSelection( meta.hasFooter() ); + wNrFooter.setText( "" + meta.getNrFooterLines() ); + wWraps.setSelection( meta.isLineWrapped() ); + wNrWraps.setText( "" + meta.getNrWraps() ); + wLayoutPaged.setSelection( meta.isLayoutPaged() ); + wNrLinesPerPage.setText( "" + meta.getNrLinesPerPage() ); + wNrLinesDocHeader.setText( "" + meta.getNrLinesDocHeader() ); + if ( meta.getFileCompression() != null ) { + wCompression.setText( meta.getFileCompression() ); + } + wNoempty.setSelection( meta.noEmptyLines() ); + wInclFilename.setSelection( meta.includeFilename() ); + wInclRownum.setSelection( meta.includeRowNumber() ); + wRownumByFile.setSelection( meta.isRowNumberByFile() ); + wDateLenient.setSelection( meta.isDateFormatLenient() ); + wAddResult.setSelection( meta.isAddResultFile() ); + + if ( meta.getFilenameField() != null ) { + wInclFilenameField.setText( meta.getFilenameField() ); + } + if ( meta.getRowNumberField() != null ) { + wInclRownumField.setText( meta.getRowNumberField() ); + } + if ( meta.getFileFormat() != null ) { + wFormat.setText( meta.getFileFormat() ); + } + wLimit.setText( "" + meta.getRowLimit() ); + + logDebug( "getting fields info..." ); + getFieldsData( meta, false ); + + if ( meta.getEncoding() != null ) { + wEncoding.setText( meta.getEncoding() ); + } + + // Error handling fields... + wErrorIgnored.setSelection( meta.isErrorIgnored() ); + wSkipBadFiles.setSelection( meta.isSkipBadFiles() ); + wSkipErrorLines.setSelection( meta.isErrorLineSkipped() ); + + if ( meta.getFileErrorField() != null ) { + wBadFileField.setText( meta.getFileErrorField() ); + } + if ( meta.getFileErrorMessageField() != null ) { + wBadFileMessageField.setText( meta.getFileErrorMessageField() ); + } + + if ( meta.getErrorCountField() != null ) { + wErrorCount.setText( meta.getErrorCountField() ); + } + if ( meta.getErrorFieldsField() != null ) { + wErrorFields.setText( meta.getErrorFieldsField() ); + } + if ( meta.getErrorTextField() != null ) { + wErrorText.setText( meta.getErrorTextField() ); + } + + if ( meta.getWarningFilesDestinationDirectory() != null ) { + wWarnDestDir.setText( meta.getWarningFilesDestinationDirectory() ); + } + if ( meta.getWarningFilesExtension() != null ) { + wWarnExt.setText( meta.getWarningFilesExtension() ); + } + + if ( meta.getErrorFilesDestinationDirectory() != null ) { + wErrorDestDir.setText( meta.getErrorFilesDestinationDirectory() ); + } + if ( meta.getErrorLineFilesExtension() != null ) { + wErrorExt.setText( meta.getErrorLineFilesExtension() ); + } + + if ( meta.getLineNumberFilesDestinationDirectory() != null ) { + wLineNrDestDir.setText( meta.getLineNumberFilesDestinationDirectory() ); + } + if ( meta.getLineNumberFilesExtension() != null ) { + wLineNrExt.setText( meta.getLineNumberFilesExtension() ); + } + + for ( int i = 0; i < meta.getFilter().length; i++ ) { + TableItem item = wFilter.table.getItem( i ); + + OldTextFileFilter filter = meta.getFilter()[ i ]; + if ( filter.getFilterString() != null ) { + item.setText( 1, filter.getFilterString() ); + } + if ( filter.getFilterPosition() >= 0 ) { + item.setText( 2, "" + filter.getFilterPosition() ); + } + item.setText( 3, filter.isFilterLastLine() ? BaseMessages.getString( PKG, "System.Combo.Yes" ) : BaseMessages + .getString( PKG, "System.Combo.No" ) ); + item.setText( 4, filter.isFilterPositive() ? BaseMessages.getString( PKG, "System.Combo.Yes" ) : BaseMessages + .getString( PKG, "System.Combo.No" ) ); + } + + // Date locale + wDateLocale.setText( meta.getDateFormatLocale().toString() ); + + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth( true ); + + wFilter.removeEmptyRows(); + wFilter.setRowNums(); + wFilter.optWidth( true ); + + if ( meta.getShortFileNameField() != null ) { + wShortFileFieldName.setText( meta.getShortFileNameField() ); + } + if ( meta.getPathField() != null ) { + wPathFieldName.setText( meta.getPathField() ); + } + if ( meta.isHiddenField() != null ) { + wIsHiddenName.setText( meta.isHiddenField() ); + } + if ( meta.getLastModificationDateField() != null ) { + wLastModificationTimeName.setText( meta.getLastModificationDateField() ); + } + if ( meta.getUriField() != null ) { + wUriName.setText( meta.getUriField() ); + } + if ( meta.getRootUriField() != null ) { + wRootUriName.setText( meta.getRootUriField() ); + } + if ( meta.getExtensionField() != null ) { + wExtensionFieldName.setText( meta.getExtensionField() ); + } + if ( meta.getSizeField() != null ) { + wSizeFieldName.setText( meta.getSizeField() ); + } + + setFlags(); + + wStepname.selectAll(); + wStepname.setFocus(); + } + + private void getFieldsData( OldTextFileInputMeta in, boolean insertAtTop ) { + for ( int i = 0; i < in.getInputFields().length; i++ ) { + TextFileInputField field = in.getInputFields()[i]; + + TableItem item; + + if ( insertAtTop ) { + item = new TableItem( wFields.table, SWT.NONE, i ); + } else { + if ( i >= wFields.table.getItemCount() ) { + item = wFields.table.getItem( i ); + } else { + item = new TableItem( wFields.table, SWT.NONE ); + } + } + + item.setText( 1, Const.NVL( field.getName(), "" ) ); + String type = field.getTypeDesc(); + String format = field.getFormat(); + String position = "" + field.getPosition(); + String length = "" + field.getLength(); + String prec = "" + field.getPrecision(); + String curr = field.getCurrencySymbol(); + String group = field.getGroupSymbol(); + String decim = field.getDecimalSymbol(); + String def = field.getNullString(); + String ifNull = field.getIfNullValue(); + String trim = field.getTrimTypeDesc(); + String rep = + field.isRepeated() ? BaseMessages.getString( PKG, "System.Combo.Yes" ) : BaseMessages.getString( PKG, + "System.Combo.No" ); + + if ( type != null ) { + item.setText( 2, type ); + } + if ( format != null ) { + item.setText( 3, format ); + } + if ( position != null && !"-1".equals( position ) ) { + item.setText( 4, position ); + } + if ( length != null && !"-1".equals( length ) ) { + item.setText( 5, length ); + } + if ( prec != null && !"-1".equals( prec ) ) { + item.setText( 6, prec ); + } + if ( curr != null ) { + item.setText( 7, curr ); + } + if ( decim != null ) { + item.setText( 8, decim ); + } + if ( group != null ) { + item.setText( 9, group ); + } + if ( def != null ) { + item.setText( 10, def ); + } + if ( ifNull != null ) { + item.setText( 11, ifNull ); + } + if ( trim != null ) { + item.setText( 12, trim ); + } + if ( rep != null ) { + item.setText( 13, rep ); + } + } + + } + + private void setEncodings() { + // Encoding of the text file: + if ( !gotEncodings ) { + gotEncodings = true; + + wEncoding.removeAll(); + List values = new ArrayList( Charset.availableCharsets().values() ); + for ( Charset charSet : values ) { + wEncoding.add( charSet.displayName() ); + } + + // Now select the default! + String defEncoding = Const.getEnvironmentVariable( "file.encoding", "UTF-8" ); + int idx = Const.indexOfString( defEncoding, wEncoding.getItems() ); + if ( idx >= 0 ) { + wEncoding.select( idx ); + } + } + } + + private void cancel() { + stepname = null; + input.setChanged( changed ); + dispose(); + } + + private void ok() { + if ( Const.isEmpty( wStepname.getText() ) ) { + return; + } + + getInfo( input ); + dispose(); + } + + private void getInfo( OldTextFileInputMeta meta ) { + stepname = wStepname.getText(); // return value + + // copy info to OldTextFileInputMeta class (input) + meta.setAcceptingFilenames( wAccFilenames.getSelection() ); + meta.setPassingThruFields( wPassThruFields.getSelection() ); + meta.setAcceptingField( wAccField.getText() ); + meta.setAcceptingStepName( wAccStep.getText() ); + meta.setAcceptingStep( transMeta.findStep( wAccStep.getText() ) ); + + meta.setFileType( wFiletype.getText() ); + meta.setFileFormat( wFormat.getText() ); + meta.setSeparator( wSeparator.getText() ); + meta.setEnclosure( wEnclosure.getText() ); + meta.setEscapeCharacter( wEscape.getText() ); + meta.setRowLimit( Const.toLong( wLimit.getText(), 0L ) ); + meta.setFilenameField( wInclFilenameField.getText() ); + meta.setRowNumberField( wInclRownumField.getText() ); + meta.setAddResultFile( wAddResult.getSelection() ); + + meta.setIncludeFilename( wInclFilename.getSelection() ); + meta.setIncludeRowNumber( wInclRownum.getSelection() ); + meta.setRowNumberByFile( wRownumByFile.getSelection() ); + meta.setHeader( wHeader.getSelection() ); + meta.setNrHeaderLines( Const.toInt( wNrHeader.getText(), 1 ) ); + meta.setFooter( wFooter.getSelection() ); + meta.setNrFooterLines( Const.toInt( wNrFooter.getText(), 1 ) ); + meta.setLineWrapped( wWraps.getSelection() ); + meta.setNrWraps( Const.toInt( wNrWraps.getText(), 1 ) ); + meta.setLayoutPaged( wLayoutPaged.getSelection() ); + meta.setNrLinesPerPage( Const.toInt( wNrLinesPerPage.getText(), 80 ) ); + meta.setNrLinesDocHeader( Const.toInt( wNrLinesDocHeader.getText(), 0 ) ); + meta.setFileCompression( wCompression.getText() ); + meta.setDateFormatLenient( wDateLenient.getSelection() ); + meta.setNoEmptyLines( wNoempty.getSelection() ); + meta.setEncoding( wEncoding.getText() ); + + int nrfiles = wFilenameList.getItemCount(); + int nrfields = wFields.nrNonEmpty(); + int nrfilters = wFilter.nrNonEmpty(); + meta.allocate( nrfiles, nrfields, nrfilters ); + + meta.setFileName( wFilenameList.getItems( 0 ) ); + meta.setFileMask( wFilenameList.getItems( 1 ) ); + meta.setExcludeFileMask( wFilenameList.getItems( 2 ) ); + meta.setFileRequired( wFilenameList.getItems( 3 ) ); + meta.setIncludeSubFolders( wFilenameList.getItems( 4 ) ); + + for ( int i = 0; i < nrfields; i++ ) { + TextFileInputField field = new TextFileInputField(); + + TableItem item = wFields.getNonEmpty( i ); + field.setName( item.getText( 1 ) ); + field.setType( ValueMeta.getType( item.getText( 2 ) ) ); + field.setFormat( item.getText( 3 ) ); + field.setPosition( Const.toInt( item.getText( 4 ), -1 ) ); + field.setLength( Const.toInt( item.getText( 5 ), -1 ) ); + field.setPrecision( Const.toInt( item.getText( 6 ), -1 ) ); + field.setCurrencySymbol( item.getText( 7 ) ); + field.setDecimalSymbol( item.getText( 8 ) ); + field.setGroupSymbol( item.getText( 9 ) ); + field.setNullString( item.getText( 10 ) ); + field.setIfNullValue( item.getText( 11 ) ); + field.setTrimType( ValueMeta.getTrimTypeByDesc( item.getText( 12 ) ) ); + field.setRepeated( BaseMessages.getString( PKG, "System.Combo.Yes" ).equalsIgnoreCase( item.getText( 13 ) ) ); + + // CHECKSTYLE:Indentation:OFF + meta.getInputFields()[i] = field; + } + + for ( int i = 0; i < nrfilters; i++ ) { + TableItem item = wFilter.getNonEmpty( i ); + OldTextFileFilter filter = new OldTextFileFilter(); + // CHECKSTYLE:Indentation:OFF + meta.getFilter()[i] = filter; + + filter.setFilterString( item.getText( 1 ) ); + filter.setFilterPosition( Const.toInt( item.getText( 2 ), -1 ) ); + filter + .setFilterLastLine( BaseMessages.getString( PKG, "System.Combo.Yes" ).equalsIgnoreCase( item.getText( 3 ) ) ); + filter + .setFilterPositive( BaseMessages.getString( PKG, "System.Combo.Yes" ).equalsIgnoreCase( item.getText( 4 ) ) ); + } + // Error handling fields... + meta.setErrorIgnored( wErrorIgnored.getSelection() ); + meta.setSkipBadFiles( wSkipBadFiles.getSelection() ); + meta.setFileErrorField( wBadFileField.getText() ); + meta.setFileErrorMessageField( wBadFileMessageField.getText() ); + meta.setErrorLineSkipped( wSkipErrorLines.getSelection() ); + meta.setErrorCountField( wErrorCount.getText() ); + meta.setErrorFieldsField( wErrorFields.getText() ); + meta.setErrorTextField( wErrorText.getText() ); + + meta.setWarningFilesDestinationDirectory( wWarnDestDir.getText() ); + meta.setWarningFilesExtension( wWarnExt.getText() ); + meta.setErrorFilesDestinationDirectory( wErrorDestDir.getText() ); + meta.setErrorLineFilesExtension( wErrorExt.getText() ); + meta.setLineNumberFilesDestinationDirectory( wLineNrDestDir.getText() ); + meta.setLineNumberFilesExtension( wLineNrExt.getText() ); + + // Date format Locale + Locale locale = EnvUtil.createLocale( wDateLocale.getText() ); + if ( !locale.equals( Locale.getDefault() ) ) { + meta.setDateFormatLocale( locale ); + } else { + meta.setDateFormatLocale( Locale.getDefault() ); + } + + meta.setShortFileNameField( wShortFileFieldName.getText() ); + meta.setPathField( wPathFieldName.getText() ); + meta.setIsHiddenField( wIsHiddenName.getText() ); + meta.setLastModificationDateField( wLastModificationTimeName.getText() ); + meta.setUriField( wUriName.getText() ); + meta.setRootUriField( wRootUriName.getText() ); + meta.setExtensionField( wExtensionFieldName.getText() ); + meta.setSizeField( wSizeFieldName.getText() ); + } + + private void get() { + if ( wFiletype.getText().equalsIgnoreCase( "CSV" ) ) { + getCSV(); + } else { + getFixed(); + } + } + + // Get the data layout + private void getCSV() { + OldTextFileInputMeta meta = new OldTextFileInputMeta(); + getInfo( meta ); + OldTextFileInputMeta previousMeta = (OldTextFileInputMeta) meta.clone(); + FileInputList textFileList = meta.getTextFileList( transMeta ); + InputStream fileInputStream; + CompressionInputStream inputStream = null; + StringBuilder lineStringBuilder = new StringBuilder( 256 ); + int fileFormatType = meta.getFileFormatTypeNr(); + + String delimiter = transMeta.environmentSubstitute( meta.getSeparator() ); + String enclosure = transMeta.environmentSubstitute( meta.getEnclosure() ); + String escapeCharacter = transMeta.environmentSubstitute( meta.getEscapeCharacter() ); + + if ( textFileList.nrOfFiles() > 0 ) { + int clearFields = meta.hasHeader() ? SWT.YES : SWT.NO; + int nrInputFields = meta.getInputFields().length; + + if ( meta.hasHeader() && nrInputFields > 0 ) { + MessageBox mb = new MessageBox( shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.ClearFieldList.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ClearFieldList.DialogTitle" ) ); + clearFields = mb.open(); + if ( clearFields == SWT.CANCEL ) { + return; + } + } + + try { + wFields.table.removeAll(); + + FileObject fileObject = textFileList.getFile( 0 ); + fileInputStream = KettleVFS.getInputStream( fileObject ); + Table table = wFields.table; + + CompressionProvider provider = + CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.getFileCompression() ); + inputStream = provider.createInputStream( fileInputStream ); + + InputStreamReader reader; + if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { + reader = new InputStreamReader( inputStream, meta.getEncoding() ); + } else { + reader = new InputStreamReader( inputStream ); + } + + OldEncodingType encodingType = OldEncodingType.guessEncodingType( reader.getEncoding() ); + + if ( clearFields == SWT.YES || !meta.hasHeader() || nrInputFields > 0 ) { + // Scan the header-line, determine fields... + String line; + + if ( meta.hasHeader() || meta.getInputFields().length == 0 ) { + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + if ( line != null ) { + // Estimate the number of input fields... + // Chop up the line using the delimiter + String[] fields = + OldTextFileInput + .guessStringsFromLine( transMeta, log, line, meta, delimiter, enclosure, escapeCharacter ); + + for ( int i = 0; i < fields.length; i++ ) { + String field = fields[i]; + if ( field == null || field.length() == 0 || ( nrInputFields == 0 && !meta.hasHeader() ) ) { + field = "Field" + ( i + 1 ); + } else { + // Trim the field + field = Const.trim( field ); + // Replace all spaces & - with underscore _ + field = Const.replace( field, " ", "_" ); + field = Const.replace( field, "-", "_" ); + } + + TableItem item = new TableItem( table, SWT.NONE ); + item.setText( 1, field ); + item.setText( 2, "String" ); // The default type is String... + } + + wFields.setRowNums(); + wFields.optWidth( true ); + + // Copy it... + getInfo( meta ); + } + } + + // Sample a few lines to determine the correct type of the fields... + String shellText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToSample.DialogTitle" ); + String lineText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToSample.DialogMessage" ); + EnterNumberDialog end = new EnterNumberDialog( shell, 100, shellText, lineText ); + int samples = end.open(); + if ( samples >= 0 ) { + getInfo( meta ); + + OldTextFileCSVImportProgressDialog pd = + new OldTextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, clearFields == SWT.YES ); + String message = pd.open(); + if ( message != null ) { + wFields.removeAll(); + + // OK, what's the result of our search? + getData( meta ); + + // If we didn't want the list to be cleared, we need to re-inject the previous values... + // + if ( clearFields == SWT.NO ) { + getFieldsData( previousMeta, true ); + wFields.table.setSelection( previousMeta.getInputFields().length, wFields.table.getItemCount() - 1 ); + } + + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth( true ); + + EnterTextDialog etd = + new EnterTextDialog( shell, BaseMessages.getString( PKG, + "TextFileInputDialog.ScanResults.DialogTitle" ), BaseMessages.getString( PKG, + "TextFileInputDialog.ScanResults.DialogMessage" ), message, true ); + etd.setReadOnly(); + etd.open(); + } + } + } else { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.UnableToReadHeaderLine.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "System.Dialog.Error.Title" ) ); + mb.open(); + } + } catch ( IOException e ) { + new ErrorDialog( shell, BaseMessages.getString( PKG, "TextFileInputDialog.IOError.DialogTitle" ), BaseMessages + .getString( PKG, "TextFileInputDialog.IOError.DialogMessage" ), e ); + } catch ( KettleException e ) { + new ErrorDialog( shell, BaseMessages.getString( PKG, "System.Dialog.Error.Title" ), BaseMessages.getString( + PKG, "TextFileInputDialog.ErrorGettingFileDesc.DialogMessage" ), e ); + } finally { + try { + if ( inputStream != null ) { + inputStream.close(); + } + } catch ( Exception e ) { + // Ignore errors + } + } + } else { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.NoValidFileFound.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "System.Dialog.Error.Title" ) ); + mb.open(); + } + } + + public static int guessPrecision( double d ) { + // Round numbers + long frac = Math.round( ( d - Math.floor( d ) ) * 1E10 ); // max precision : 10 + int precision = 10; + + // 0,34 --> 3400000000 + // 0 to the right --> precision -1! + // 0 to the right means frac%10 == 0 + + while ( precision >= 0 && ( frac % 10 ) == 0 ) { + frac /= 10; + precision--; + } + precision++; + + return precision; + } + + public static int guessIntLength( double d ) { + double flr = Math.floor( d ); + int len = 1; + + while ( flr > 9 ) { + flr /= 10; + flr = Math.floor( flr ); + len++; + } + + return len; + } + + // Preview the data + private void preview() { + // Create the XML input step + OldTextFileInputMeta oneMeta = new OldTextFileInputMeta(); + getInfo( oneMeta ); + + if ( oneMeta.isAcceptingFilenames() ) { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_INFORMATION ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.Dialog.SpecifyASampleFile.Message" ) ); + mb.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Dialog.SpecifyASampleFile.Title" ) ); + mb.open(); + return; + } + + TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, + wStepname.getText() ); + + EnterNumberDialog numberDialog = + new EnterNumberDialog( shell, props.getDefaultPreviewSize(), BaseMessages.getString( PKG, + "TextFileInputDialog.PreviewSize.DialogTitle" ), BaseMessages.getString( PKG, + "TextFileInputDialog.PreviewSize.DialogMessage" ) ); + int previewSize = numberDialog.open(); + if ( previewSize > 0 ) { + TransPreviewProgressDialog progressDialog = + new TransPreviewProgressDialog( shell, previewMeta, new String[] { wStepname.getText() }, + new int[] { previewSize } ); + progressDialog.open(); + + Trans trans = progressDialog.getTrans(); + String loggingText = progressDialog.getLoggingText(); + + if ( !progressDialog.isCancelled() ) { + if ( trans.getResult() != null && trans.getResult().getNrErrors() > 0 ) { + EnterTextDialog etd = + new EnterTextDialog( shell, BaseMessages.getString( PKG, "System.Dialog.PreviewError.Title" ), + BaseMessages.getString( PKG, "System.Dialog.PreviewError.Message" ), loggingText, true ); + etd.setReadOnly(); + etd.open(); + } + } + + PreviewRowsDialog prd = + new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog + .getPreviewRowsMeta( wStepname.getText() ), progressDialog.getPreviewRows( wStepname.getText() ), + loggingText ); + prd.open(); + } + } + + // Get the first x lines + private void first( boolean skipHeaders ) { + OldTextFileInputMeta info = new OldTextFileInputMeta(); + getInfo( info ); + + try { + if ( info.getTextFileList( transMeta ).nrOfFiles() > 0 ) { + String shellText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogTitle" ); + String lineText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogMessage" ); + EnterNumberDialog end = new EnterNumberDialog( shell, 100, shellText, lineText ); + int nrLines = end.open(); + if ( nrLines >= 0 ) { + List linesList = getFirst( nrLines, skipHeaders ); + if ( linesList != null && linesList.size() > 0 ) { + String firstlines = ""; + for ( String aLinesList : linesList ) { + firstlines += aLinesList + Const.CR; + } + EnterTextDialog etd = + new EnterTextDialog( shell, BaseMessages.getString( PKG, + "TextFileInputDialog.ContentOfFirstFile.DialogTitle" ), ( nrLines == 0 ? BaseMessages.getString( + PKG, "TextFileInputDialog.ContentOfFirstFile.AllLines.DialogMessage" ) : BaseMessages.getString( + PKG, "TextFileInputDialog.ContentOfFirstFile.NLines.DialogMessage", "" + nrLines ) ), firstlines, + true ); + etd.setReadOnly(); + etd.open(); + } else { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.UnableToReadLines.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "TextFileInputDialog.UnableToReadLines.DialogTitle" ) ); + mb.open(); + } + } + } else { + MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); + mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.NoValidFile.DialogMessage" ) ); + mb.setText( BaseMessages.getString( PKG, "System.Dialog.Error.Title" ) ); + mb.open(); + } + } catch ( KettleException e ) { + new ErrorDialog( shell, BaseMessages.getString( PKG, "System.Dialog.Error.Title" ), BaseMessages.getString( PKG, + "TextFileInputDialog.ErrorGettingData.DialogMessage" ), e ); + } + } + + // Get the first x lines + private List getFirst( int nrlines, boolean skipHeaders ) throws KettleException { + OldTextFileInputMeta meta = new OldTextFileInputMeta(); + getInfo( meta ); + FileInputList textFileList = meta.getTextFileList( transMeta ); + + InputStream fi; + CompressionInputStream f = null; + StringBuilder lineStringBuilder = new StringBuilder( 256 ); + int fileFormatType = meta.getFileFormatTypeNr(); + + List retval = new ArrayList(); + + if ( textFileList.nrOfFiles() > 0 ) { + FileObject file = textFileList.getFile( 0 ); + try { + fi = KettleVFS.getInputStream( file ); + + CompressionProvider provider = + CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.getFileCompression() ); + f = provider.createInputStream( fi ); + + InputStreamReader reader; + if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { + reader = new InputStreamReader( f, meta.getEncoding() ); + } else { + reader = new InputStreamReader( f ); + } + OldEncodingType encodingType = OldEncodingType.guessEncodingType( reader.getEncoding() ); + + int linenr = 0; + int maxnr = nrlines + ( meta.hasHeader() ? meta.getNrHeaderLines() : 0 ); + + if ( skipHeaders ) { + // Skip the header lines first if more then one, it helps us position + if ( meta.isLayoutPaged() && meta.getNrLinesDocHeader() > 0 ) { + int skipped = 0; + String line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + while ( line != null && skipped < meta.getNrLinesDocHeader() - 1 ) { + skipped++; + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + } + } + + // Skip the header lines first if more then one, it helps us position + if ( meta.hasHeader() && meta.getNrHeaderLines() > 0 ) { + int skipped = 0; + String line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + while ( line != null && skipped < meta.getNrHeaderLines() - 1 ) { + skipped++; + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + } + } + } + + String line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + while ( line != null && ( linenr < maxnr || nrlines == 0 ) ) { + retval.add( line ); + linenr++; + line = OldTextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + } + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "TextFileInputDialog.Exception.ErrorGettingFirstLines", + "" + nrlines, file.getName().getURI() ), e ); + } finally { + try { + if ( f != null ) { + f.close(); + } + } catch ( Exception e ) { + // Ignore errors + } + } + } + + return retval; + } + + private void getFixed() { + OldTextFileInputMeta info = new OldTextFileInputMeta(); + getInfo( info ); + + Shell sh = new Shell( shell, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN ); + + try { + List rows = getFirst( 50, false ); + fields = getFields( info, rows ); + + final OldTextFileImportWizardPage1 page1 = new OldTextFileImportWizardPage1( "1", props, rows, fields ); + page1.createControl( sh ); + final OldTextFileImportWizardPage2 page2 = new OldTextFileImportWizardPage2( "2", props, rows, fields ); + page2.createControl( sh ); + + Wizard wizard = new Wizard() { + public boolean performFinish() { + wFields.clearAll( false ); + + for ( TextFileInputFieldInterface field1 : fields ) { + TextFileInputField field = (TextFileInputField) field1; + if ( !field.isIgnored() && field.getLength() > 0 ) { + TableItem item = new TableItem( wFields.table, SWT.NONE ); + item.setText( 1, field.getName() ); + item.setText( 2, "" + field.getTypeDesc() ); + item.setText( 3, "" + field.getFormat() ); + item.setText( 4, "" + field.getPosition() ); + item.setText( 5, field.getLength() < 0 ? "" : "" + field.getLength() ); + item.setText( 6, field.getPrecision() < 0 ? "" : "" + field.getPrecision() ); + item.setText( 7, "" + field.getCurrencySymbol() ); + item.setText( 8, "" + field.getDecimalSymbol() ); + item.setText( 9, "" + field.getGroupSymbol() ); + item.setText( 10, "" + field.getNullString() ); + item.setText( 11, "" + field.getIfNullValue() ); + item.setText( 12, "" + field.getTrimTypeDesc() ); + item.setText( 13, field.isRepeated() ? BaseMessages.getString( PKG, "System.Combo.Yes" ) : BaseMessages + .getString( PKG, "System.Combo.No" ) ); + } + + } + int size = wFields.table.getItemCount(); + if ( size == 0 ) { + new TableItem( wFields.table, SWT.NONE ); + } + + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth( true ); + + input.setChanged(); + + return true; + } + }; + + wizard.addPage( page1 ); + wizard.addPage( page2 ); + + WizardDialog wd = new WizardDialog( shell, wizard ); + WizardDialog.setDefaultImage( GUIResource.getInstance().getImageWizard() ); + wd.setMinimumPageSize( 700, 375 ); + wd.updateSize(); + wd.open(); + } catch ( Exception e ) { + new ErrorDialog( shell, BaseMessages.getString( PKG, "TextFileInputDialog.ErrorShowingFixedWizard.DialogTitle" ), + BaseMessages.getString( PKG, "TextFileInputDialog.ErrorShowingFixedWizard.DialogMessage" ), e ); + } + } + + private Vector getFields( OldTextFileInputMeta info, List rows ) { + Vector fields = new Vector(); + + int maxsize = 0; + for ( String row : rows ) { + int len = row.length(); + if ( len > maxsize ) { + maxsize = len; + } + } + + int prevEnd = 0; + int dummynr = 1; + + for ( int i = 0; i < info.getInputFields().length; i++ ) { + TextFileInputField f = info.getInputFields()[i]; + + // See if positions are skipped, if this is the case, add dummy fields... + if ( f.getPosition() != prevEnd ) { // gap + + TextFileInputField field = new TextFileInputField( "Dummy" + dummynr, prevEnd, f.getPosition() - prevEnd ); + field.setIgnored( true ); // don't include in result by default. + fields.add( field ); + dummynr++; + } + + TextFileInputField field = new TextFileInputField( f.getName(), f.getPosition(), f.getLength() ); + field.setType( f.getType() ); + field.setIgnored( false ); + field.setFormat( f.getFormat() ); + field.setPrecision( f.getPrecision() ); + field.setTrimType( f.getTrimType() ); + field.setDecimalSymbol( f.getDecimalSymbol() ); + field.setGroupSymbol( f.getGroupSymbol() ); + field.setCurrencySymbol( f.getCurrencySymbol() ); + field.setRepeated( f.isRepeated() ); + field.setNullString( f.getNullString() ); + + fields.add( field ); + + prevEnd = field.getPosition() + field.getLength(); + } + + if ( info.getInputFields().length == 0 ) { + TextFileInputField field = new TextFileInputField( "Field1", 0, maxsize ); + fields.add( field ); + } else { + // Take the last field and see if it reached until the maximum... + TextFileInputField f = info.getInputFields()[info.getInputFields().length - 1]; + + int pos = f.getPosition(); + int len = f.getLength(); + if ( pos + len < maxsize ) { + // If not, add an extra trailing field! + TextFileInputField field = new TextFileInputField( "Dummy" + dummynr, pos + len, maxsize - pos - len ); + field.setIgnored( true ); // don't include in result by default. + fields.add( field ); + } + } + + Collections.sort( fields ); + + return fields; + } + + private void addAdditionalFieldsTab() { + // //////////////////////// + // START OF ADDITIONAL FIELDS TAB /// + // //////////////////////// + wAdditionalFieldsTab = new CTabItem( wTabFolder, SWT.NONE ); + wAdditionalFieldsTab.setText( BaseMessages.getString( PKG, "TextFileInputDialog.AdditionalFieldsTab.TabTitle" ) ); + + wAdditionalFieldsComp = new Composite( wTabFolder, SWT.NONE ); + props.setLook( wAdditionalFieldsComp ); + + FormLayout fieldsLayout = new FormLayout(); + fieldsLayout.marginWidth = 3; + fieldsLayout.marginHeight = 3; + wAdditionalFieldsComp.setLayout( fieldsLayout ); + + // ShortFileFieldName line + wlShortFileFieldName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlShortFileFieldName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ShortFileFieldName.Label" ) ); + props.setLook( wlShortFileFieldName ); + fdlShortFileFieldName = new FormData(); + fdlShortFileFieldName.left = new FormAttachment( 0, 0 ); + fdlShortFileFieldName.top = new FormAttachment( margin, margin ); + fdlShortFileFieldName.right = new FormAttachment( middle, -margin ); + wlShortFileFieldName.setLayoutData( fdlShortFileFieldName ); + + wShortFileFieldName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wShortFileFieldName ); + wShortFileFieldName.addModifyListener( lsMod ); + fdShortFileFieldName = new FormData(); + fdShortFileFieldName.left = new FormAttachment( middle, 0 ); + fdShortFileFieldName.right = new FormAttachment( 100, -margin ); + fdShortFileFieldName.top = new FormAttachment( margin, margin ); + wShortFileFieldName.setLayoutData( fdShortFileFieldName ); + + // ExtensionFieldName line + wlExtensionFieldName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlExtensionFieldName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ExtensionFieldName.Label" ) ); + props.setLook( wlExtensionFieldName ); + fdlExtensionFieldName = new FormData(); + fdlExtensionFieldName.left = new FormAttachment( 0, 0 ); + fdlExtensionFieldName.top = new FormAttachment( wShortFileFieldName, margin ); + fdlExtensionFieldName.right = new FormAttachment( middle, -margin ); + wlExtensionFieldName.setLayoutData( fdlExtensionFieldName ); + + wExtensionFieldName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wExtensionFieldName ); + wExtensionFieldName.addModifyListener( lsMod ); + fdExtensionFieldName = new FormData(); + fdExtensionFieldName.left = new FormAttachment( middle, 0 ); + fdExtensionFieldName.right = new FormAttachment( 100, -margin ); + fdExtensionFieldName.top = new FormAttachment( wShortFileFieldName, margin ); + wExtensionFieldName.setLayoutData( fdExtensionFieldName ); + + // PathFieldName line + wlPathFieldName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlPathFieldName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.PathFieldName.Label" ) ); + props.setLook( wlPathFieldName ); + fdlPathFieldName = new FormData(); + fdlPathFieldName.left = new FormAttachment( 0, 0 ); + fdlPathFieldName.top = new FormAttachment( wExtensionFieldName, margin ); + fdlPathFieldName.right = new FormAttachment( middle, -margin ); + wlPathFieldName.setLayoutData( fdlPathFieldName ); + + wPathFieldName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wPathFieldName ); + wPathFieldName.addModifyListener( lsMod ); + fdPathFieldName = new FormData(); + fdPathFieldName.left = new FormAttachment( middle, 0 ); + fdPathFieldName.right = new FormAttachment( 100, -margin ); + fdPathFieldName.top = new FormAttachment( wExtensionFieldName, margin ); + wPathFieldName.setLayoutData( fdPathFieldName ); + + // SizeFieldName line + wlSizeFieldName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlSizeFieldName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.SizeFieldName.Label" ) ); + props.setLook( wlSizeFieldName ); + fdlSizeFieldName = new FormData(); + fdlSizeFieldName.left = new FormAttachment( 0, 0 ); + fdlSizeFieldName.top = new FormAttachment( wPathFieldName, margin ); + fdlSizeFieldName.right = new FormAttachment( middle, -margin ); + wlSizeFieldName.setLayoutData( fdlSizeFieldName ); + + wSizeFieldName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wSizeFieldName ); + wSizeFieldName.addModifyListener( lsMod ); + fdSizeFieldName = new FormData(); + fdSizeFieldName.left = new FormAttachment( middle, 0 ); + fdSizeFieldName.right = new FormAttachment( 100, -margin ); + fdSizeFieldName.top = new FormAttachment( wPathFieldName, margin ); + wSizeFieldName.setLayoutData( fdSizeFieldName ); + + // IsHiddenName line + wlIsHiddenName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlIsHiddenName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.IsHiddenName.Label" ) ); + props.setLook( wlIsHiddenName ); + fdlIsHiddenName = new FormData(); + fdlIsHiddenName.left = new FormAttachment( 0, 0 ); + fdlIsHiddenName.top = new FormAttachment( wSizeFieldName, margin ); + fdlIsHiddenName.right = new FormAttachment( middle, -margin ); + wlIsHiddenName.setLayoutData( fdlIsHiddenName ); + + wIsHiddenName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wIsHiddenName ); + wIsHiddenName.addModifyListener( lsMod ); + fdIsHiddenName = new FormData(); + fdIsHiddenName.left = new FormAttachment( middle, 0 ); + fdIsHiddenName.right = new FormAttachment( 100, -margin ); + fdIsHiddenName.top = new FormAttachment( wSizeFieldName, margin ); + wIsHiddenName.setLayoutData( fdIsHiddenName ); + + // LastModificationTimeName line + wlLastModificationTimeName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlLastModificationTimeName.setText( BaseMessages.getString( PKG, + "TextFileInputDialog.LastModificationTimeName.Label" ) ); + props.setLook( wlLastModificationTimeName ); + fdlLastModificationTimeName = new FormData(); + fdlLastModificationTimeName.left = new FormAttachment( 0, 0 ); + fdlLastModificationTimeName.top = new FormAttachment( wIsHiddenName, margin ); + fdlLastModificationTimeName.right = new FormAttachment( middle, -margin ); + wlLastModificationTimeName.setLayoutData( fdlLastModificationTimeName ); + + wLastModificationTimeName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wLastModificationTimeName ); + wLastModificationTimeName.addModifyListener( lsMod ); + fdLastModificationTimeName = new FormData(); + fdLastModificationTimeName.left = new FormAttachment( middle, 0 ); + fdLastModificationTimeName.right = new FormAttachment( 100, -margin ); + fdLastModificationTimeName.top = new FormAttachment( wIsHiddenName, margin ); + wLastModificationTimeName.setLayoutData( fdLastModificationTimeName ); + + // UriName line + wlUriName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlUriName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.UriName.Label" ) ); + props.setLook( wlUriName ); + fdlUriName = new FormData(); + fdlUriName.left = new FormAttachment( 0, 0 ); + fdlUriName.top = new FormAttachment( wLastModificationTimeName, margin ); + fdlUriName.right = new FormAttachment( middle, -margin ); + wlUriName.setLayoutData( fdlUriName ); + + wUriName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wUriName ); + wUriName.addModifyListener( lsMod ); + fdUriName = new FormData(); + fdUriName.left = new FormAttachment( middle, 0 ); + fdUriName.right = new FormAttachment( 100, -margin ); + fdUriName.top = new FormAttachment( wLastModificationTimeName, margin ); + wUriName.setLayoutData( fdUriName ); + + // RootUriName line + wlRootUriName = new Label( wAdditionalFieldsComp, SWT.RIGHT ); + wlRootUriName.setText( BaseMessages.getString( PKG, "TextFileInputDialog.RootUriName.Label" ) ); + props.setLook( wlRootUriName ); + fdlRootUriName = new FormData(); + fdlRootUriName.left = new FormAttachment( 0, 0 ); + fdlRootUriName.top = new FormAttachment( wUriName, margin ); + fdlRootUriName.right = new FormAttachment( middle, -margin ); + wlRootUriName.setLayoutData( fdlRootUriName ); + + wRootUriName = new TextVar( transMeta, wAdditionalFieldsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wRootUriName ); + wRootUriName.addModifyListener( lsMod ); + fdRootUriName = new FormData(); + fdRootUriName.left = new FormAttachment( middle, 0 ); + fdRootUriName.right = new FormAttachment( 100, -margin ); + fdRootUriName.top = new FormAttachment( wUriName, margin ); + wRootUriName.setLayoutData( fdRootUriName ); + + fdAdditionalFieldsComp = new FormData(); + fdAdditionalFieldsComp.left = new FormAttachment( 0, 0 ); + fdAdditionalFieldsComp.top = new FormAttachment( 0, 0 ); + fdAdditionalFieldsComp.right = new FormAttachment( 100, 0 ); + fdAdditionalFieldsComp.bottom = new FormAttachment( 100, 0 ); + wAdditionalFieldsComp.setLayoutData( fdAdditionalFieldsComp ); + + wAdditionalFieldsComp.layout(); + wAdditionalFieldsTab.setControl( wAdditionalFieldsComp ); + + // /////////////////////////////////////////////////////////// + // / END OF ADDITIONAL FIELDS TAB + // /////////////////////////////////////////////////////////// + + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldVariableButtonListenerFactory.java b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldVariableButtonListenerFactory.java new file mode 100644 index 000000000000..04264fc4d327 --- /dev/null +++ b/ui/src/org/pentaho/di/ui/trans/steps/oldtextfileinput/OldVariableButtonListenerFactory.java @@ -0,0 +1,91 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.ui.trans.steps.oldtextfileinput; + +import java.util.Arrays; + +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Text; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.ui.core.dialog.EnterSelectionDialog; +import org.pentaho.di.ui.core.widget.GetCaretPositionInterface; +import org.pentaho.di.ui.core.widget.InsertTextInterface; + +public class OldVariableButtonListenerFactory { + private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! + + // Listen to the Variable... button + public static final SelectionAdapter getSelectionAdapter( final Composite composite, final Text destination, + VariableSpace space ) { + return getSelectionAdapter( composite, destination, null, null, space ); + } + + // Listen to the Variable... button + public static final SelectionAdapter getSelectionAdapter( final Composite composite, final Text destination, + final GetCaretPositionInterface getCaretPositionInterface, final InsertTextInterface insertTextInterface, + final VariableSpace space ) { + return new SelectionAdapter() { + public void widgetSelected( SelectionEvent e ) { + String[] keys = space.listVariables(); + Arrays.sort( keys ); + + int size = keys.length; + String[] key = new String[size]; + String[] val = new String[size]; + String[] str = new String[size]; + + for ( int i = 0; i < keys.length; i++ ) { + key[i] = keys[i]; + val[i] = space.getVariable( key[i] ); + str[i] = key[i] + " [" + val[i] + "]"; + } + + // Before focus is lost, we get the position of where the selected variable needs to be inserted. + int position = 0; + if ( getCaretPositionInterface != null ) { + position = getCaretPositionInterface.getCaretPosition(); + } + + EnterSelectionDialog esd = new EnterSelectionDialog( composite.getShell(), str, + BaseMessages.getString( PKG, "System.Dialog.SelectEnvironmentVar.Title" ), + BaseMessages.getString( PKG, "System.Dialog.SelectEnvironmentVar.Message" ) ); + if ( esd.open() != null ) { + int nr = esd.getSelectionNr(); + String var = "${" + key[nr] + "}"; + + if ( insertTextInterface == null ) { + destination.insert( var ); + // destination.setToolTipText(StringUtil.environmentSubstitute( destination.getText() ) ); + e.doit = false; + } else { + insertTextInterface.insertText( var, position ); + } + } + } + }; + } +} diff --git a/ui/src/org/pentaho/di/ui/trans/steps/parallelgzipcsv/ParGzipCsvInputDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/parallelgzipcsv/ParGzipCsvInputDialog.java index dcf22e2e4a76..3a16e2c1957d 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/parallelgzipcsv/ParGzipCsvInputDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/parallelgzipcsv/ParGzipCsvInputDialog.java @@ -75,6 +75,7 @@ import org.pentaho.di.trans.steps.textfileinput.TextFileInput; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputUtils; import org.pentaho.di.ui.core.dialog.EnterNumberDialog; import org.pentaho.di.ui.core.dialog.EnterTextDialog; import org.pentaho.di.ui.core.dialog.ErrorDialog; @@ -86,6 +87,7 @@ import org.pentaho.di.ui.core.widget.TextVar; import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog; import org.pentaho.di.ui.trans.step.BaseStepDialog; +import org.pentaho.di.ui.trans.steps.oldtextfileinput.OldTextFileCSVImportProgressDialog; import org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog; public class ParGzipCsvInputDialog extends BaseStepDialog implements StepDialogInterface { @@ -782,7 +784,7 @@ private void getCSV() { // Read a line of data to determine the number of rows... // String line = - TextFileInput.getLine( + TextFileInputUtils.getLine( log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder( 1000 ) ); // Split the string, header or data into parts... @@ -835,8 +837,8 @@ private void getCSV() { if ( samples >= 0 ) { getInfo( meta ); - TextFileCSVImportProgressDialog pd = - new TextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, true ); + OldTextFileCSVImportProgressDialog pd = + new OldTextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, true ); String message = pd.open(); if ( message != null ) { wFields.removeAll(); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileCSVImportProgressDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileCSVImportProgressDialog.java index 5032def35c13..79c5c54b9a91 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileCSVImportProgressDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileCSVImportProgressDialog.java @@ -47,11 +47,13 @@ import org.pentaho.di.core.util.StringEvaluator; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.steps.baseinput.BaseInputStepMeta; import org.pentaho.di.trans.steps.textfileinput.EncodingType; -import org.pentaho.di.trans.steps.textfileinput.InputFileMetaInterface; import org.pentaho.di.trans.steps.textfileinput.TextFileInput; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputReader; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputUtils; import org.pentaho.di.trans.steps.textfileinput.TextFileLine; import org.pentaho.di.ui.core.dialog.ErrorDialog; @@ -64,10 +66,10 @@ */ public class TextFileCSVImportProgressDialog { private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! - + private Shell shell; - private InputFileMetaInterface meta; + private TextFileInputMeta meta; private int samples; @@ -91,7 +93,7 @@ public class TextFileCSVImportProgressDialog { * Creates a new dialog that will handle the wait while we're finding out what tables, views etc we can reach in the * database. */ - public TextFileCSVImportProgressDialog( Shell shell, InputFileMetaInterface meta, TransMeta transMeta, + public TextFileCSVImportProgressDialog( Shell shell, TextFileInputMeta meta, TransMeta transMeta, InputStreamReader reader, int samples, boolean replaceMeta ) { this.shell = shell; this.meta = meta; @@ -153,7 +155,7 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { DecimalFormatSymbols dfs = new DecimalFormatSymbols(); - int nrfields = meta.getInputFields().length; + int nrfields = meta.inputFiles.inputFields.length; RowMetaInterface outputRowMeta = new RowMeta(); meta.getFields( outputRowMeta, null, null, null, transMeta, null, null ); @@ -193,7 +195,7 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { int[][] numberLength = new int[nrfields][Const.getNumberFormats().length]; // remember the length? for ( int i = 0; i < nrfields; i++ ) { - TextFileInputField field = meta.getInputFields()[i]; + TextFileInputField field = meta.inputFiles.inputFields[i]; if ( log.isDebug() ) { debug = "init field #" + i; @@ -201,8 +203,8 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { if ( replaceMeta ) { // Clear previous info... - field.setName( meta.getInputFields()[i].getName() ); - field.setType( meta.getInputFields()[i].getType() ); + field.setName( meta.inputFiles.inputFields[i].getName() ); + field.setType( meta.inputFiles.inputFields[i].getType() ); field.setFormat( "" ); field.setLength( -1 ); field.setPrecision( -1 ); @@ -239,9 +241,9 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { numberFormatCount[i] = Const.getNumberFormats().length; } - InputFileMetaInterface strinfo = (InputFileMetaInterface) meta.clone(); + TextFileInputMeta strinfo = (TextFileInputMeta) meta.clone(); for ( int i = 0; i < nrfields; i++ ) { - strinfo.getInputFields()[i].setType( ValueMetaInterface.TYPE_STRING ); + strinfo.inputFiles.inputFields[i].setType( ValueMetaInterface.TYPE_STRING ); } // Sample rows... @@ -254,14 +256,14 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { // However, if it doesn't have a header, take a new line // - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); fileLineNumber++; int skipped = 1; - if ( meta.hasHeader() ) { + if ( meta.content.header ) { - while ( line != null && skipped < meta.getNrHeaderLines() ) { - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + while ( line != null && skipped < meta.content.nrHeaderLines ) { + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); skipped++; fileLineNumber++; } @@ -294,15 +296,14 @@ private String doScan( IProgressMonitor monitor ) throws KettleException { valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } - String delimiter = transMeta.environmentSubstitute( meta.getSeparator() ); - String enclosure = transMeta.environmentSubstitute( meta.getEnclosure() ); - String escapeCharacter = transMeta.environmentSubstitute( meta.getEscapeCharacter() ); + String delimiter = transMeta.environmentSubstitute( meta.content.separator ); + String enclosure = transMeta.environmentSubstitute( meta.content.enclosure ); + String escapeCharacter = transMeta.environmentSubstitute( meta.content.escapeCharacter ); Object[] r = - TextFileInput.convertLineToRow( - log, new TextFileLine( line, fileLineNumber, null ), strinfo, null, 0, outputRowMeta, - convertRowMeta, meta.getFilePaths( transMeta )[0], rownumber, delimiter, enclosure, escapeCharacter, - null, false, false, false, false, false, false, false, false, null, null, false, null, null, null, - null, 0 ); + TextFileInputUtils.convertLineToRow( log, new TextFileLine( line, fileLineNumber, null ), strinfo, null, 0, + outputRowMeta, convertRowMeta, meta.getFilePaths( transMeta )[0], rownumber, delimiter, enclosure, + escapeCharacter, null, new BaseInputStepMeta.AdditionalOutputFields(), null, null, false, null, null, + null, null, 0 ); if ( r == null ) { errorFound = true; @@ -333,7 +334,7 @@ log, new TextFileLine( line, fileLineNumber, null ), strinfo, null, 0, outputRow // Grab another line... // - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineBuffer ); } monitor.worked( 1 ); @@ -347,7 +348,7 @@ log, new TextFileLine( line, fileLineNumber, null ), strinfo, null, 0, outputRow message.append( BaseMessages.getString( PKG, "TextFileCSVImportProgressDialog.Info.HorizontalLine" ) ); for ( int i = 0; i < nrfields; i++ ) { - TextFileInputField field = meta.getInputFields()[i]; + TextFileInputField field = meta.inputFiles.inputFields[i]; StringEvaluator evaluator = evaluators.get( i ); List evaluationResults = evaluator.getStringEvaluationResults(); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage1.java b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage1.java index ec5f30c9d485..b56f55fddd54 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage1.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage1.java @@ -39,6 +39,7 @@ import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; import org.pentaho.di.ui.core.PropsUI; import org.pentaho.di.ui.core.widget.TableDraw; +import org.pentaho.di.ui.core.widget.OldTableDraw; public class TextFileImportWizardPage1 extends WizardPage // implements Listener { diff --git a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage2.java b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage2.java index 4462b3620069..b0cb066d48c0 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage2.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileImportWizardPage2.java @@ -44,10 +44,13 @@ import org.pentaho.di.core.Const; import org.pentaho.di.core.Props; import org.pentaho.di.core.gui.TextFileInputFieldInterface; +import org.pentaho.di.core.gui.TextFileInputFieldInterface; import org.pentaho.di.core.row.ValueMeta; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; import org.pentaho.di.ui.core.PropsUI; public class TextFileImportWizardPage2 extends WizardPage { diff --git a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java index efeb4e486824..bf8cddfb740c 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java @@ -23,6 +23,16 @@ package org.pentaho.di.ui.trans.steps.textfileinput; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Vector; + import org.apache.commons.vfs2.FileObject; import org.eclipse.jface.wizard.Wizard; import org.eclipse.jface.wizard.WizardDialog; @@ -77,11 +87,14 @@ import org.pentaho.di.trans.step.BaseStepMeta; import org.pentaho.di.trans.step.StepDialogInterface; import org.pentaho.di.trans.step.StepMeta; -import org.pentaho.di.trans.steps.textfileinput.EncodingType; import org.pentaho.di.trans.steps.textfileinput.TextFileFilter; import org.pentaho.di.trans.steps.textfileinput.TextFileInput; import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputReader; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputUtils; +import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta; +import org.pentaho.di.trans.steps.textfileinput.EncodingType; import org.pentaho.di.ui.core.dialog.EnterNumberDialog; import org.pentaho.di.ui.core.dialog.EnterSelectionDialog; import org.pentaho.di.ui.core.dialog.EnterTextDialog; @@ -94,16 +107,6 @@ import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog; import org.pentaho.di.ui.trans.step.BaseStepDialog; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Vector; - public class TextFileInputDialog extends BaseStepDialog implements StepDialogInterface { private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! @@ -574,7 +577,7 @@ public void widgetDefaultSelected( SelectionEvent e ) { SelectionAdapter selA = new SelectionAdapter() { public void widgetSelected( SelectionEvent arg0 ) { wFilenameList.add( wFilename.getText(), wFilemask.getText(), wExcludeFilemask.getText(), - TextFileInputMeta.RequiredFilesCode[ 0 ], TextFileInputMeta.RequiredFilesCode[ 0 ] ); + TextFileInputMeta.RequiredFilesCode[ 0 ], TextFileInputMeta.RequiredFilesCode[ 0 ] ); wFilename.setText( "" ); wFilemask.setText( "" ); wExcludeFilemask.setText( "" ); @@ -2040,7 +2043,7 @@ private void addFieldsTabs() { fdGet.bottom = new FormAttachment( 100, 0 ); wGet.setLayoutData( fdGet ); - final int FieldsRows = input.getInputFields().length; + final int FieldsRows = input.inputFiles.inputFields.length; ColumnInfo[] colinf = new ColumnInfo[] { @@ -2183,10 +2186,10 @@ public void setFlags() { * The TextFileInputMeta object to obtain the data from. */ public void getData( TextFileInputMeta meta ) { - wAccFilenames.setSelection( meta.isAcceptingFilenames() ); - wPassThruFields.setSelection( meta.isPassingThruFields() ); - if ( meta.getAcceptingField() != null ) { - wAccField.setText( meta.getAcceptingField() ); + wAccFilenames.setSelection( meta.inputFiles.acceptingFilenames ); + wPassThruFields.setSelection( meta.inputFiles.passingThruFields ); + if ( meta.inputFiles.acceptingField != null ) { + wAccField.setText( meta.inputFiles.acceptingField ); } if ( meta.getAcceptingStep() != null ) { wAccStep.setText( meta.getAcceptingStep().getName() ); @@ -2197,55 +2200,55 @@ public void getData( TextFileInputMeta meta ) { for ( int i = 0; i < meta.getFileName().length; i++ ) { wFilenameList - .add( meta.getFileName()[ i ], meta.getFileMask()[ i ], meta.getExludeFileMask()[ i ], - meta.getRequiredFilesDesc( meta.getFileRequired()[ i ] ), - meta.getRequiredFilesDesc( meta.getIncludeSubFolders()[ i ] ) ); + .add( meta.getFileName()[ i ], meta.inputFiles.fileMask[ i ], meta.inputFiles.excludeFileMask[ i ], + meta.getRequiredFilesDesc( meta.inputFiles.fileRequired[ i ] ), + meta.getRequiredFilesDesc( meta.inputFiles.includeSubFolders[ i ] ) ); } wFilenameList.removeEmptyRows(); wFilenameList.setRowNums(); wFilenameList.optWidth( true ); } - if ( meta.getFileType() != null ) { - wFiletype.setText( meta.getFileType() ); + if ( meta.content.fileType != null ) { + wFiletype.setText( meta.content.fileType ); } - if ( meta.getSeparator() != null ) { - wSeparator.setText( meta.getSeparator() ); + if ( meta.content.separator != null ) { + wSeparator.setText( meta.content.separator ); } - if ( meta.getEnclosure() != null ) { - wEnclosure.setText( meta.getEnclosure() ); + if ( meta.content.enclosure != null ) { + wEnclosure.setText( meta.content.enclosure ); } - if ( meta.getEscapeCharacter() != null ) { - wEscape.setText( meta.getEscapeCharacter() ); + if ( meta.content.escapeCharacter != null ) { + wEscape.setText( meta.content.escapeCharacter ); } - wHeader.setSelection( meta.hasHeader() ); - wNrHeader.setText( "" + meta.getNrHeaderLines() ); - wFooter.setSelection( meta.hasFooter() ); - wNrFooter.setText( "" + meta.getNrFooterLines() ); - wWraps.setSelection( meta.isLineWrapped() ); - wNrWraps.setText( "" + meta.getNrWraps() ); - wLayoutPaged.setSelection( meta.isLayoutPaged() ); - wNrLinesPerPage.setText( "" + meta.getNrLinesPerPage() ); - wNrLinesDocHeader.setText( "" + meta.getNrLinesDocHeader() ); - if ( meta.getFileCompression() != null ) { - wCompression.setText( meta.getFileCompression() ); + wHeader.setSelection( meta.content.header ); + wNrHeader.setText( "" + meta.content.nrHeaderLines); + wFooter.setSelection( meta.content.footer ); + wNrFooter.setText( "" + meta.content.nrFooterLines ); + wWraps.setSelection( meta.content.lineWrapped ); + wNrWraps.setText( "" + meta.content.nrWraps ); + wLayoutPaged.setSelection( meta.content.layoutPaged); + wNrLinesPerPage.setText( "" + meta.content.nrLinesPerPage ); + wNrLinesDocHeader.setText( "" + meta.content.nrLinesDocHeader ); + if ( meta.content.fileCompression != null ) { + wCompression.setText( meta.content.fileCompression ); } - wNoempty.setSelection( meta.noEmptyLines() ); - wInclFilename.setSelection( meta.includeFilename() ); - wInclRownum.setSelection( meta.includeRowNumber() ); - wRownumByFile.setSelection( meta.isRowNumberByFile() ); - wDateLenient.setSelection( meta.isDateFormatLenient() ); - wAddResult.setSelection( meta.isAddResultFile() ); - - if ( meta.getFilenameField() != null ) { - wInclFilenameField.setText( meta.getFilenameField() ); + wNoempty.setSelection( meta.content.noEmptyLines ); + wInclFilename.setSelection( meta.content.includeFilename ); + wInclRownum.setSelection( meta.content.includeRowNumber ); + wRownumByFile.setSelection( meta.content.rowNumberByFile ); + wDateLenient.setSelection( meta.content.dateFormatLenient ); + wAddResult.setSelection( meta.inputFiles.isaddresult ); + + if ( meta.content.filenameField != null ) { + wInclFilenameField.setText( meta.content.filenameField ); } - if ( meta.getRowNumberField() != null ) { - wInclRownumField.setText( meta.getRowNumberField() ); + if ( meta.content.rowNumberField != null ) { + wInclRownumField.setText( meta.content.rowNumberField ); } - if ( meta.getFileFormat() != null ) { - wFormat.setText( meta.getFileFormat() ); + if ( meta.content.fileFormat != null ) { + wFormat.setText( meta.content.fileFormat); } - wLimit.setText( "" + meta.getRowLimit() ); + wLimit.setText( "" + meta.content.rowLimit); logDebug( "getting fields info..." ); getFieldsData( meta, false ); @@ -2255,15 +2258,15 @@ public void getData( TextFileInputMeta meta ) { } // Error handling fields... - wErrorIgnored.setSelection( meta.isErrorIgnored() ); - wSkipBadFiles.setSelection( meta.isSkipBadFiles() ); + wErrorIgnored.setSelection( meta.errorHandling.errorIgnored ); + wSkipBadFiles.setSelection( meta.errorHandling.skipBadFiles ); wSkipErrorLines.setSelection( meta.isErrorLineSkipped() ); - if ( meta.getFileErrorField() != null ) { - wBadFileField.setText( meta.getFileErrorField() ); + if ( meta.errorHandling.fileErrorField != null ) { + wBadFileField.setText( meta.errorHandling.fileErrorField ); } - if ( meta.getFileErrorMessageField() != null ) { - wBadFileMessageField.setText( meta.getFileErrorMessageField() ); + if ( meta.errorHandling.fileErrorMessageField != null ) { + wBadFileMessageField.setText( meta.errorHandling.fileErrorMessageField ); } if ( meta.getErrorCountField() != null ) { @@ -2276,25 +2279,25 @@ public void getData( TextFileInputMeta meta ) { wErrorText.setText( meta.getErrorTextField() ); } - if ( meta.getWarningFilesDestinationDirectory() != null ) { - wWarnDestDir.setText( meta.getWarningFilesDestinationDirectory() ); + if ( meta.errorHandling.warningFilesDestinationDirectory != null ) { + wWarnDestDir.setText( meta.errorHandling.warningFilesDestinationDirectory ); } - if ( meta.getWarningFilesExtension() != null ) { - wWarnExt.setText( meta.getWarningFilesExtension() ); + if ( meta.errorHandling.warningFilesExtension != null ) { + wWarnExt.setText( meta.errorHandling.warningFilesExtension ); } - if ( meta.getErrorFilesDestinationDirectory() != null ) { - wErrorDestDir.setText( meta.getErrorFilesDestinationDirectory() ); + if ( meta.errorHandling.errorFilesDestinationDirectory != null ) { + wErrorDestDir.setText( meta.errorHandling.errorFilesDestinationDirectory ); } - if ( meta.getErrorLineFilesExtension() != null ) { - wErrorExt.setText( meta.getErrorLineFilesExtension() ); + if ( meta.errorHandling.errorFilesExtension != null ) { + wErrorExt.setText( meta.errorHandling.errorFilesExtension ); } - if ( meta.getLineNumberFilesDestinationDirectory() != null ) { - wLineNrDestDir.setText( meta.getLineNumberFilesDestinationDirectory() ); + if ( meta.errorHandling.lineNumberFilesDestinationDirectory != null ) { + wLineNrDestDir.setText( meta.errorHandling.lineNumberFilesDestinationDirectory ); } - if ( meta.getLineNumberFilesExtension() != null ) { - wLineNrExt.setText( meta.getLineNumberFilesExtension() ); + if ( meta.errorHandling.lineNumberFilesExtension != null ) { + wLineNrExt.setText( meta.errorHandling.lineNumberFilesExtension ); } for ( int i = 0; i < meta.getFilter().length; i++ ) { @@ -2314,7 +2317,7 @@ public void getData( TextFileInputMeta meta ) { } // Date locale - wDateLocale.setText( meta.getDateFormatLocale().toString() ); + wDateLocale.setText( meta.content.dateFormatLocale.toString() ); wFields.removeEmptyRows(); wFields.setRowNums(); @@ -2324,29 +2327,29 @@ public void getData( TextFileInputMeta meta ) { wFilter.setRowNums(); wFilter.optWidth( true ); - if ( meta.getShortFileNameField() != null ) { - wShortFileFieldName.setText( meta.getShortFileNameField() ); + if ( meta.additionalOutputFields.shortFilenameField != null ) { + wShortFileFieldName.setText( meta.additionalOutputFields.shortFilenameField ); } - if ( meta.getPathField() != null ) { - wPathFieldName.setText( meta.getPathField() ); + if ( meta.additionalOutputFields.pathField != null ) { + wPathFieldName.setText( meta.additionalOutputFields.pathField ); } - if ( meta.isHiddenField() != null ) { - wIsHiddenName.setText( meta.isHiddenField() ); + if ( meta.additionalOutputFields.hiddenField != null ) { + wIsHiddenName.setText( meta.additionalOutputFields.hiddenField ); } - if ( meta.getLastModificationDateField() != null ) { - wLastModificationTimeName.setText( meta.getLastModificationDateField() ); + if ( meta.additionalOutputFields.lastModificationField != null ) { + wLastModificationTimeName.setText( meta.additionalOutputFields.lastModificationField ); } - if ( meta.getUriField() != null ) { - wUriName.setText( meta.getUriField() ); + if ( meta.additionalOutputFields.uriField != null ) { + wUriName.setText( meta.additionalOutputFields.uriField ); } - if ( meta.getRootUriField() != null ) { - wRootUriName.setText( meta.getRootUriField() ); + if ( meta.additionalOutputFields.rootUriField != null ) { + wRootUriName.setText( meta.additionalOutputFields.rootUriField ); } - if ( meta.getExtensionField() != null ) { - wExtensionFieldName.setText( meta.getExtensionField() ); + if ( meta.additionalOutputFields.extensionField != null ) { + wExtensionFieldName.setText( meta.additionalOutputFields.extensionField ); } - if ( meta.getSizeField() != null ) { - wSizeFieldName.setText( meta.getSizeField() ); + if ( meta.additionalOutputFields.sizeField != null ) { + wSizeFieldName.setText( meta.additionalOutputFields.sizeField ); } setFlags(); @@ -2356,8 +2359,8 @@ public void getData( TextFileInputMeta meta ) { } private void getFieldsData( TextFileInputMeta in, boolean insertAtTop ) { - for ( int i = 0; i < in.getInputFields().length; i++ ) { - TextFileInputField field = in.getInputFields()[i]; + for ( int i = 0; i < in.inputFiles.inputFields.length; i++ ) { + TextFileInputField field = in.inputFiles.inputFields[i]; TableItem item; @@ -2466,38 +2469,38 @@ private void getInfo( TextFileInputMeta meta ) { stepname = wStepname.getText(); // return value // copy info to TextFileInputMeta class (input) - meta.setAcceptingFilenames( wAccFilenames.getSelection() ); - meta.setPassingThruFields( wPassThruFields.getSelection() ); - meta.setAcceptingField( wAccField.getText() ); - meta.setAcceptingStepName( wAccStep.getText() ); + meta.inputFiles.acceptingFilenames= wAccFilenames.getSelection() ; + meta.inputFiles.passingThruFields= wPassThruFields.getSelection() ; + meta.inputFiles.acceptingField= wAccField.getText() ; + meta.inputFiles.acceptingStepName= wAccStep.getText() ; meta.setAcceptingStep( transMeta.findStep( wAccStep.getText() ) ); - meta.setFileType( wFiletype.getText() ); - meta.setFileFormat( wFormat.getText() ); - meta.setSeparator( wSeparator.getText() ); - meta.setEnclosure( wEnclosure.getText() ); - meta.setEscapeCharacter( wEscape.getText() ); - meta.setRowLimit( Const.toLong( wLimit.getText(), 0L ) ); - meta.setFilenameField( wInclFilenameField.getText() ); - meta.setRowNumberField( wInclRownumField.getText() ); - meta.setAddResultFile( wAddResult.getSelection() ); - - meta.setIncludeFilename( wInclFilename.getSelection() ); - meta.setIncludeRowNumber( wInclRownum.getSelection() ); - meta.setRowNumberByFile( wRownumByFile.getSelection() ); - meta.setHeader( wHeader.getSelection() ); - meta.setNrHeaderLines( Const.toInt( wNrHeader.getText(), 1 ) ); - meta.setFooter( wFooter.getSelection() ); - meta.setNrFooterLines( Const.toInt( wNrFooter.getText(), 1 ) ); - meta.setLineWrapped( wWraps.getSelection() ); - meta.setNrWraps( Const.toInt( wNrWraps.getText(), 1 ) ); - meta.setLayoutPaged( wLayoutPaged.getSelection() ); - meta.setNrLinesPerPage( Const.toInt( wNrLinesPerPage.getText(), 80 ) ); - meta.setNrLinesDocHeader( Const.toInt( wNrLinesDocHeader.getText(), 0 ) ); - meta.setFileCompression( wCompression.getText() ); - meta.setDateFormatLenient( wDateLenient.getSelection() ); - meta.setNoEmptyLines( wNoempty.getSelection() ); - meta.setEncoding( wEncoding.getText() ); + meta.content.fileType = wFiletype.getText() ; + meta.content.fileFormat = wFormat.getText() ; + meta.content.separator = wSeparator.getText() ; + meta.content.enclosure = wEnclosure.getText() ; + meta.content.escapeCharacter = wEscape.getText() ; + meta.content.rowLimit = Const.toLong( wLimit.getText(), 0L ) ; + meta.content.filenameField = wInclFilenameField.getText() ; + meta.content.rowNumberField = wInclRownumField.getText() ; + meta.inputFiles.isaddresult= wAddResult.getSelection() ; + + meta.content.includeFilename = wInclFilename.getSelection() ; + meta.content.includeRowNumber = wInclRownum.getSelection() ; + meta.content.rowNumberByFile = wRownumByFile.getSelection() ; + meta.content.header = wHeader.getSelection() ; + meta.content.nrHeaderLines = Const.toInt( wNrHeader.getText(), 1 ) ; + meta.content.footer = wFooter.getSelection() ; + meta.content.nrFooterLines = Const.toInt( wNrFooter.getText(), 1 ) ; + meta.content.lineWrapped = wWraps.getSelection() ; + meta.content.nrWraps = Const.toInt( wNrWraps.getText(), 1 ) ; + meta.content.layoutPaged = wLayoutPaged.getSelection() ; + meta.content.nrLinesPerPage= Const.toInt( wNrLinesPerPage.getText(), 80 ) ; + meta.content.nrLinesDocHeader = Const.toInt( wNrLinesDocHeader.getText(), 0 ) ; + meta.content.fileCompression = wCompression.getText() ; + meta.content.dateFormatLenient = wDateLenient.getSelection() ; + meta.content.noEmptyLines = wNoempty.getSelection() ; + meta.content.encoding = wEncoding.getText() ; int nrfiles = wFilenameList.getItemCount(); int nrfields = wFields.nrNonEmpty(); @@ -2505,10 +2508,10 @@ private void getInfo( TextFileInputMeta meta ) { meta.allocate( nrfiles, nrfields, nrfilters ); meta.setFileName( wFilenameList.getItems( 0 ) ); - meta.setFileMask( wFilenameList.getItems( 1 ) ); - meta.setExcludeFileMask( wFilenameList.getItems( 2 ) ); - meta.setFileRequired( wFilenameList.getItems( 3 ) ); - meta.setIncludeSubFolders( wFilenameList.getItems( 4 ) ); + meta.inputFiles.fileMask= wFilenameList.getItems( 1 ) ; + meta.inputFiles.excludeFileMask= wFilenameList.getItems( 2 ) ; + meta.inputFiles_fileRequired( wFilenameList.getItems( 3 ) ); + meta.inputFiles_includeSubFolders( wFilenameList.getItems( 4 ) ); for ( int i = 0; i < nrfields; i++ ) { TextFileInputField field = new TextFileInputField(); @@ -2529,7 +2532,7 @@ private void getInfo( TextFileInputMeta meta ) { field.setRepeated( BaseMessages.getString( PKG, "System.Combo.Yes" ).equalsIgnoreCase( item.getText( 13 ) ) ); // CHECKSTYLE:Indentation:OFF - meta.getInputFields()[i] = field; + meta.inputFiles.inputFields[i] = field; } for ( int i = 0; i < nrfilters; i++ ) { @@ -2546,38 +2549,38 @@ private void getInfo( TextFileInputMeta meta ) { .setFilterPositive( BaseMessages.getString( PKG, "System.Combo.Yes" ).equalsIgnoreCase( item.getText( 4 ) ) ); } // Error handling fields... - meta.setErrorIgnored( wErrorIgnored.getSelection() ); - meta.setSkipBadFiles( wSkipBadFiles.getSelection() ); - meta.setFileErrorField( wBadFileField.getText() ); - meta.setFileErrorMessageField( wBadFileMessageField.getText() ); + meta.errorHandling.errorIgnored= wErrorIgnored.getSelection() ; + meta.errorHandling.skipBadFiles= wSkipBadFiles.getSelection() ; + meta.errorHandling.fileErrorField= wBadFileField.getText() ; + meta.errorHandling.fileErrorMessageField= wBadFileMessageField.getText() ; meta.setErrorLineSkipped( wSkipErrorLines.getSelection() ); meta.setErrorCountField( wErrorCount.getText() ); meta.setErrorFieldsField( wErrorFields.getText() ); meta.setErrorTextField( wErrorText.getText() ); - meta.setWarningFilesDestinationDirectory( wWarnDestDir.getText() ); - meta.setWarningFilesExtension( wWarnExt.getText() ); - meta.setErrorFilesDestinationDirectory( wErrorDestDir.getText() ); - meta.setErrorLineFilesExtension( wErrorExt.getText() ); - meta.setLineNumberFilesDestinationDirectory( wLineNrDestDir.getText() ); - meta.setLineNumberFilesExtension( wLineNrExt.getText() ); + meta.errorHandling.warningFilesDestinationDirectory= wWarnDestDir.getText() ; + meta.errorHandling.warningFilesExtension= wWarnExt.getText() ; + meta.errorHandling.errorFilesDestinationDirectory= wErrorDestDir.getText() ; + meta.errorHandling.errorFilesExtension=wErrorExt.getText() ; + meta.errorHandling.lineNumberFilesDestinationDirectory= wLineNrDestDir.getText() ; + meta.errorHandling.lineNumberFilesExtension= wLineNrExt.getText() ; // Date format Locale Locale locale = EnvUtil.createLocale( wDateLocale.getText() ); if ( !locale.equals( Locale.getDefault() ) ) { - meta.setDateFormatLocale( locale ); + meta.content.dateFormatLocale = locale ; } else { - meta.setDateFormatLocale( Locale.getDefault() ); + meta.content.dateFormatLocale = Locale.getDefault() ; } - meta.setShortFileNameField( wShortFileFieldName.getText() ); - meta.setPathField( wPathFieldName.getText() ); - meta.setIsHiddenField( wIsHiddenName.getText() ); - meta.setLastModificationDateField( wLastModificationTimeName.getText() ); - meta.setUriField( wUriName.getText() ); - meta.setRootUriField( wRootUriName.getText() ); - meta.setExtensionField( wExtensionFieldName.getText() ); - meta.setSizeField( wSizeFieldName.getText() ); + meta.additionalOutputFields.shortFilenameField=wShortFileFieldName.getText() ; + meta.additionalOutputFields.pathField=wPathFieldName.getText() ; + meta.additionalOutputFields.hiddenField=wIsHiddenName.getText() ; + meta.additionalOutputFields.lastModificationField=wLastModificationTimeName.getText() ; + meta.additionalOutputFields.uriField=wUriName.getText() ; + meta.additionalOutputFields.rootUriField=wRootUriName.getText() ; + meta.additionalOutputFields.extensionField=wExtensionFieldName.getText() ; + meta.additionalOutputFields.sizeField=wSizeFieldName.getText() ; } private void get() { @@ -2599,15 +2602,15 @@ private void getCSV() { StringBuilder lineStringBuilder = new StringBuilder( 256 ); int fileFormatType = meta.getFileFormatTypeNr(); - String delimiter = transMeta.environmentSubstitute( meta.getSeparator() ); - String enclosure = transMeta.environmentSubstitute( meta.getEnclosure() ); - String escapeCharacter = transMeta.environmentSubstitute( meta.getEscapeCharacter() ); + String delimiter = transMeta.environmentSubstitute( meta.content.separator ); + String enclosure = transMeta.environmentSubstitute( meta.content.enclosure ); + String escapeCharacter = transMeta.environmentSubstitute( meta.content.escapeCharacter ); if ( textFileList.nrOfFiles() > 0 ) { - int clearFields = meta.hasHeader() ? SWT.YES : SWT.NO; - int nrInputFields = meta.getInputFields().length; + int clearFields = meta.content.header ? SWT.YES : SWT.NO; + int nrInputFields = meta.inputFiles.inputFields.length; - if ( meta.hasHeader() && nrInputFields > 0 ) { + if ( meta.content.header && nrInputFields > 0 ) { MessageBox mb = new MessageBox( shell, SWT.YES | SWT.NO | SWT.CANCEL | SWT.ICON_QUESTION ); mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.ClearFieldList.DialogMessage" ) ); mb.setText( BaseMessages.getString( PKG, "TextFileInputDialog.ClearFieldList.DialogTitle" ) ); @@ -2625,7 +2628,7 @@ private void getCSV() { Table table = wFields.table; CompressionProvider provider = - CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.getFileCompression() ); + CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.content.fileCompression ); inputStream = provider.createInputStream( fileInputStream ); InputStreamReader reader; @@ -2637,22 +2640,22 @@ private void getCSV() { EncodingType encodingType = EncodingType.guessEncodingType( reader.getEncoding() ); - if ( clearFields == SWT.YES || !meta.hasHeader() || nrInputFields > 0 ) { + if ( clearFields == SWT.YES || !meta.content.header || nrInputFields > 0 ) { // Scan the header-line, determine fields... String line; - if ( meta.hasHeader() || meta.getInputFields().length == 0 ) { - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + if ( meta.content.header || meta.inputFiles.inputFields.length == 0 ) { + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); if ( line != null ) { // Estimate the number of input fields... // Chop up the line using the delimiter String[] fields = - TextFileInput + TextFileInputUtils .guessStringsFromLine( transMeta, log, line, meta, delimiter, enclosure, escapeCharacter ); for ( int i = 0; i < fields.length; i++ ) { String field = fields[i]; - if ( field == null || field.length() == 0 || ( nrInputFields == 0 && !meta.hasHeader() ) ) { + if ( field == null || field.length() == 0 || ( nrInputFields == 0 && !meta.content.header ) ) { field = "Field" + ( i + 1 ); } else { // Trim the field @@ -2696,7 +2699,7 @@ private void getCSV() { // if ( clearFields == SWT.NO ) { getFieldsData( previousMeta, true ); - wFields.table.setSelection( previousMeta.getInputFields().length, wFields.table.getItemCount() - 1 ); + wFields.table.setSelection( previousMeta.inputFiles.inputFields.length, wFields.table.getItemCount() - 1 ); } wFields.removeEmptyRows(); @@ -2777,7 +2780,7 @@ private void preview() { TextFileInputMeta oneMeta = new TextFileInputMeta(); getInfo( oneMeta ); - if ( oneMeta.isAcceptingFilenames() ) { + if ( oneMeta.inputFiles.acceptingFilenames ) { MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_INFORMATION ); mb.setMessage( BaseMessages.getString( PKG, "TextFileInputDialog.Dialog.SpecifyASampleFile.Message" ) ); mb.setText( BaseMessages.getString( PKG, "TextFileInputDialog.Dialog.SpecifyASampleFile.Title" ) ); @@ -2884,7 +2887,7 @@ private List getFirst( int nrlines, boolean skipHeaders ) throws KettleE fi = KettleVFS.getInputStream( file ); CompressionProvider provider = - CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.getFileCompression() ); + CompressionProviderFactory.getInstance().createCompressionProviderInstance( meta.content.fileCompression ); f = provider.createInputStream( fi ); InputStreamReader reader; @@ -2896,35 +2899,35 @@ private List getFirst( int nrlines, boolean skipHeaders ) throws KettleE EncodingType encodingType = EncodingType.guessEncodingType( reader.getEncoding() ); int linenr = 0; - int maxnr = nrlines + ( meta.hasHeader() ? meta.getNrHeaderLines() : 0 ); + int maxnr = nrlines + ( meta.content.header ? meta.content.nrHeaderLines : 0 ); if ( skipHeaders ) { // Skip the header lines first if more then one, it helps us position - if ( meta.isLayoutPaged() && meta.getNrLinesDocHeader() > 0 ) { + if ( meta.content.layoutPaged && meta.content.nrLinesDocHeader > 0 ) { int skipped = 0; - String line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); - while ( line != null && skipped < meta.getNrLinesDocHeader() - 1 ) { + String line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + while ( line != null && skipped < meta.content.nrLinesDocHeader - 1 ) { skipped++; - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); } } // Skip the header lines first if more then one, it helps us position - if ( meta.hasHeader() && meta.getNrHeaderLines() > 0 ) { + if ( meta.content.header && meta.content.nrHeaderLines > 0 ) { int skipped = 0; - String line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); - while ( line != null && skipped < meta.getNrHeaderLines() - 1 ) { + String line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + while ( line != null && skipped < meta.content.nrHeaderLines - 1 ) { skipped++; - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); } } } - String line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + String line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); while ( line != null && ( linenr < maxnr || nrlines == 0 ) ) { retval.add( line ); linenr++; - line = TextFileInput.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); + line = TextFileInputUtils.getLine( log, reader, encodingType, fileFormatType, lineStringBuilder ); } } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "TextFileInputDialog.Exception.ErrorGettingFirstLines", @@ -3026,8 +3029,8 @@ private Vector getFields( TextFileInputMeta info, L int prevEnd = 0; int dummynr = 1; - for ( int i = 0; i < info.getInputFields().length; i++ ) { - TextFileInputField f = info.getInputFields()[i]; + for ( int i = 0; i < info.inputFiles.inputFields.length; i++ ) { + TextFileInputField f = info.inputFiles.inputFields[i]; // See if positions are skipped, if this is the case, add dummy fields... if ( f.getPosition() != prevEnd ) { // gap @@ -3055,12 +3058,12 @@ private Vector getFields( TextFileInputMeta info, L prevEnd = field.getPosition() + field.getLength(); } - if ( info.getInputFields().length == 0 ) { + if ( info.inputFiles.inputFields.length == 0 ) { TextFileInputField field = new TextFileInputField( "Field1", 0, maxsize ); fields.add( field ); } else { // Take the last field and see if it reached until the maximum... - TextFileInputField f = info.getInputFields()[info.getInputFields().length - 1]; + TextFileInputField f = info.inputFiles.inputFields[info.inputFiles.inputFields.length - 1]; int pos = f.getPosition(); int len = f.getLength();