From 5bb0aa331349c09584590ebddf57859ae1ba70e9 Mon Sep 17 00:00:00 2001 From: lance Date: Mon, 6 Apr 2026 12:42:51 +0800 Subject: [PATCH] Improve YamlInput component spacing, add unit tests, and fix Sonar issues Signed-off-by: lance --- .../transforms/datasets/golden-yaml-input.csv | 3 +- .../transforms/yamlinput/YamlInput.java | 165 +++--- .../transforms/yamlinput/YamlInputData.java | 13 +- .../transforms/yamlinput/YamlInputDialog.java | 286 +++++----- .../transforms/yamlinput/YamlInputField.java | 7 +- .../transforms/yamlinput/YamlInputMeta.java | 3 +- .../transforms/yamlinput/YamlReader.java | 513 ++++++++---------- .../messages/messages_en_US.properties | 2 +- .../yamlinput/YamlInputDataTests.java | 51 ++ .../yamlinput/YamlInputFieldTests.java | 119 ++++ .../yamlinput/YamlInputMetaTest.java | 218 ++++++++ .../transforms/yamlinput/YamlInputTests.java | 296 ++++++++++ .../transforms/yamlinput/YamlReaderTests.java | 230 ++++++++ 13 files changed, 1363 insertions(+), 543 deletions(-) create mode 100644 plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDataTests.java create mode 100644 plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputFieldTests.java create mode 100644 plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java create mode 100644 plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java diff --git a/integration-tests/transforms/datasets/golden-yaml-input.csv b/integration-tests/transforms/datasets/golden-yaml-input.csv index cd156b7622d..6a7bac46011 100644 --- a/integration-tests/transforms/datasets/golden-yaml-input.csv +++ b/integration-tests/transforms/datasets/golden-yaml-input.csv @@ -1,3 +1,2 @@ organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics -The Apache Software Foundation,The ASF,Apache Hop,4447,Y,"[Spark, Flink, Beam, Commons, VFS] -","{number-of-libs: 1159,number-of-messages-bundles: 13025,integration-test-projects: {count: 39,pipelines: 1032,workflows: 498}}" +The Apache Software Foundation,The ASF,Apache Hop,4447,Y,"[Spark, Flink, Beam, Commons, VFS]","{number-of-libs: 1159,number-of-messages-bundles: 13025,integration-test-projects: {count: 39,pipelines: 1032,workflows: 498}}" diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java index 34179a1978e..6b48d39a897 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java @@ -53,9 +53,9 @@ public YamlInput( } private void handleMissingFiles() throws HopException { - List nonExistantFiles = data.files.getNonExistentFiles(); - if (!nonExistantFiles.isEmpty()) { - String message = FileInputList.getRequiredFilesDescription(nonExistantFiles); + List nonExistentFiles = data.files.getNonExistentFiles(); + if (!nonExistentFiles.isEmpty()) { + String message = FileInputList.getRequiredFilesDescription(nonExistentFiles); logError( BaseMessages.getString(PKG, "YamlInput.Log.RequiredFilesTitle"), BaseMessages.getString(PKG, "YamlInput.Log.RequiredFiles", message)); @@ -78,9 +78,10 @@ private void handleMissingFiles() throws HopException { private boolean readNextString() { try { - data.readrow = getRow(); // Grab another row ... + // Grab another row ... + data.readRow = getRow(); - if (data.readrow == null) { + if (data.readRow == null) { // finished processing! if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "YamlInput.Log.FinishedProcessing")); @@ -116,14 +117,13 @@ private boolean readNextString() { } // get field value - String fieldvalue = getInputRowMeta().getString(data.readrow, data.indexOfYamlField); - + String fieldValue = getInputRowMeta().getString(data.readRow, data.indexOfYamlField); getLinesInput(); if (isDetailed()) { logDetailed( BaseMessages.getString( - PKG, "YamlInput.Log.YAMLStream", meta.getYamlField(), fieldvalue)); + PKG, "YamlInput.Log.YAMLStream", meta.getYamlField(), fieldValue)); } if (meta.isSourceFile()) { @@ -131,15 +131,14 @@ private boolean readNextString() { // source is a file. data.yaml = new YamlReader(); - data.yaml.loadFile(HopVfs.getFileObject(fieldvalue, variables)); + data.yaml.loadFile(HopVfs.getFileObject(fieldValue, variables)); dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + data.yaml.getBytesReadFromFile(); - addFileToResultFilesname(data.yaml.getFile()); - + addFileToResultFilesName(data.yaml.getFile()); } else { data.yaml = new YamlReader(); - data.yaml.loadString(fieldvalue); + data.yaml.loadString(fieldValue); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "YamlInput.Log.UnexpectedError", e.toString())); @@ -151,7 +150,7 @@ private boolean readNextString() { return false; } - private void addFileToResultFilesname(FileObject file) { + private void addFileToResultFilesName(FileObject file) { if (meta.isAddingResultFile()) { // Add this to the result file names... ResultFile resultFile = @@ -164,7 +163,7 @@ private void addFileToResultFilesname(FileObject file) { private boolean openNextFile() { try { - if (data.filenr >= data.files.nrOfFiles()) { + if (data.fileIndex >= data.files.nrOfFiles()) { // finished processing! if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "YamlInput.Log.FinishedProcessing")); @@ -172,10 +171,10 @@ private boolean openNextFile() { return false; } // Get file to process from list - data.file = data.files.getFile(data.filenr); + data.file = data.files.getFile(data.fileIndex); // Move file pointer ahead! - data.filenr++; + data.fileIndex++; if (meta.isIgnoringEmptyFile() && data.file.getContent().getSize() == 0) { if (isBasic()) { @@ -198,7 +197,7 @@ private boolean openNextFile() { dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + data.yaml.getBytesReadFromFile(); - addFileToResultFilesname(data.file); + addFileToResultFilesName(data.file); if (isDetailed()) { logDetailed( @@ -210,7 +209,7 @@ private boolean openNextFile() { BaseMessages.getString( PKG, "YamlInput.Log.UnableToOpenFile", - "" + data.filenr, + "" + data.fileIndex, data.file.toString(), e.toString())); stopAll(); @@ -243,24 +242,25 @@ public boolean processRow() throws HopException { } // Grab a row Object[] r = getOneRow(); - - if (r == null) { - setOutputDone(); // signal end to receiver(s) - return false; // end of data or error. + if (Utils.isEmpty(r)) { + // signal end to receiver(s) + setOutputDone(); + // end of data or error. + return false; } if (isRowLevel()) { logRowlevel( BaseMessages.getString(PKG, "YamlInput.Log.ReadRow", data.outputRowMeta.getString(r))); } - incrementLinesOutput(); - data.rownr++; + data.rowIndex++; Object[] rowCopy = data.outputRowMeta.cloneRow(r); - putRow(data.outputRowMeta, rowCopy); // copy row to output rowset(s) + // copy row to output rowset(s) + putRow(data.outputRowMeta, rowCopy); - if (meta.getRowLimit() > 0 && data.rownr > meta.getRowLimit()) { + if (meta.getRowLimit() > 0 && data.rowIndex > meta.getRowLimit()) { // limit has been reached: stop now. setOutputDone(); return false; @@ -269,71 +269,52 @@ public boolean processRow() throws HopException { } private Object[] getOneRow() throws HopException { - Object[] row = null; - boolean rowAvailable = false; - boolean fileOpened = false; if (!meta.isInFields()) { - while (data.file == null || !fileOpened && !rowAvailable) { - if (data.file != null) { - // We have opened a file - // read one row - row = getRowData(); - - if (row == null) { - // No row extracted - // let's see for the next file - if (!openNextFile()) { - return null; - } - fileOpened = true; - } else { - // We had extracted one row - rowAvailable = true; - } - } else { - // First time we get there - // we have to open a new file - if (!openNextFile()) { - return null; - } - fileOpened = true; - } + return getOneRowFromFileMode(); + } + + return getOneRowFromFieldMode(); + } + + private Object[] getOneRowFromFileMode() throws HopException { + while (true) { + if (data.file == null && !openNextFile()) { + return new Object[0]; } - } else { - while (data.readrow == null || (data.readrow != null && !fileOpened && !rowAvailable)) { - if (data.readrow != null) { - // We have red the incoming Yaml value - // let's get one row - row = getRowData(); - if (row == null) { - // No row.. reader next row - if (readNextString()) { - return null; - } - fileOpened = true; - } else { - // We have returned one row - rowAvailable = true; - } - } else { - // First time we get there - // We have to parse incoming Yaml value - if (readNextString()) { - return null; - } - fileOpened = true; - } - if (data.readrow == null) { - return null; - } + + if (data.file == null) { + continue; + } + + Object[] row = getRowData(); + if (row != null && row.length > 0) { + return row; } - } - if (!rowAvailable) { - row = getRowData(); + if (!openNextFile()) { + return new Object[0]; + } } + } + + private Object[] getOneRowFromFieldMode() throws HopException { + while (true) { + if (data.readRow == null) { + if (readNextString()) { + return new Object[0]; + } + continue; + } + + Object[] row = getRowData(); + if (row != null && row.length > 0) { + return row; + } - return row; + if (readNextString()) { + return new Object[0]; + } + } } private Object[] getRowData() throws HopException { @@ -343,13 +324,13 @@ private Object[] getRowData() throws HopException { try { // Create new row... outputRowData = data.yaml.getRow(data.rowMeta); - if (outputRowData == null) { - return null; + if (outputRowData == null || outputRowData.length == 0) { + return new Object[0]; } - if (data.readrow != null) { + if (data.readRow != null) { outputRowData = - RowDataUtil.addRowData(data.readrow, data.totalPreviousFields, outputRowData); + RowDataUtil.addRowData(data.readRow, data.totalPreviousFields, outputRowData); } else { outputRowData = RowDataUtil.resizeArray(outputRowData, data.totalOutStreamFields); } @@ -362,9 +343,8 @@ private Object[] getRowData() throws HopException { } // See if we need to add the row number to the row... if (meta.isIncludeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) { - outputRowData[rowIndex++] = data.rownr; + outputRowData[rowIndex] = data.rowIndex; } - } catch (Exception e) { boolean sendToErrorRow = false; String errorMessage = null; @@ -390,9 +370,8 @@ private Object[] getRowData() throws HopException { @Override public boolean init() { - if (super.init()) { - data.rownr = 1L; + data.rowIndex = 1L; data.nrInputFields = meta.getInputFields().size(); data.rowMeta = new RowMeta(); diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java index 41eb050e754..0b53fbad962 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputData.java @@ -21,14 +21,13 @@ import org.apache.hop.core.fileinput.FileInputList; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.pipeline.transform.BaseTransformData; -import org.apache.hop.pipeline.transform.ITransformData; @SuppressWarnings("java:S1104") -public class YamlInputData extends BaseTransformData implements ITransformData { +public class YamlInputData extends BaseTransformData { public IRowMeta outputRowMeta; public int nrInputFields; - public Object[] readrow; + public Object[] readRow; public int totalPreviousFields; public int totalOutFields; public int totalOutStreamFields; @@ -37,9 +36,9 @@ public class YamlInputData extends BaseTransformData implements ITransformData { public FileInputList files; public FileObject file; - public int filenr; + public int fileIndex; - public long rownr; + public long rowIndex; public int indexOfYamlField; public YamlReader yaml; @@ -49,10 +48,10 @@ public class YamlInputData extends BaseTransformData implements ITransformData { public YamlInputData() { super(); - this.filenr = 0; + this.fileIndex = 0; this.indexOfYamlField = -1; this.nrInputFields = -1; - this.readrow = null; + this.readRow = null; this.totalPreviousFields = 0; this.file = null; this.totalOutFields = 0; diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java index 6ab30addf7b..8dbf1b40fbb 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java @@ -17,7 +17,6 @@ package org.apache.hop.pipeline.transforms.yamlinput; -import java.util.ArrayList; import org.apache.hop.core.Const; import org.apache.hop.core.Props; import org.apache.hop.core.exception.HopException; @@ -74,21 +73,22 @@ public class YamlInputDialog extends BaseTransformDialog { public static final String CONST_YAML_INPUT_DIALOG_ERROR_PARSING_DATA_DIALOG_MESSAGE = "YamlInputDialog.ErrorParsingData.DialogMessage"; - private static final String YES = "Y"; - - public static final String[] NO_YES_DEC = + protected static final String[] NO_YES_DEC = new String[] { BaseMessages.getString(PKG, "System.Combo.No"), BaseMessages.getString(PKG, "System.Combo.Yes") }; - public static final String[] RequiredFilesCode = new String[] {"N", "Y"}; private Label wlFilename; private Label wlYamlIsAFile; - private Button wbbFilename; // Browse: add file or directory - private Button wbdFilename; // Delete - private Button wbeFilename; // Edit - private Button wbaFilename; // Add or change + // Browse: add file or directory + private Button wbbFilename; + // Delete + private Button wbdFilename; + // Edit + private Button wbeFilename; + // Add or change + private Button wbaFilename; private TextVar wFilename; private Label wlFilenameList; @@ -111,10 +111,10 @@ public class YamlInputDialog extends BaseTransformDialog { private Label wlInclFilenameField; private TextVar wInclFilenameField; private Label wlAddResult; - private Button wInclRownum; + private Button wInclRowNum; - private Label wlInclRownumField; - private TextVar wInclRownumField; + private Label wlInclRowNumField; + private TextVar wInclRowNumField; private Label wlLimit; private Text wLimit; @@ -129,12 +129,7 @@ public class YamlInputDialog extends BaseTransformDialog { private Label wlDoNotFailIfNoFile; private Button wDoNotFailIfNoFile; - private YamlInputMeta input; - - public static final int[] dateLengths = new int[] {23, 19, 14, 10, 10, 10, 10, 8, 8, 8, 8, 6, 6}; - - ArrayList listpath = new ArrayList<>(); - String precNodeName = null; + private final YamlInputMeta input; public YamlInputDialog( Shell parent, IVariables variables, YamlInputMeta transformMeta, PipelineMeta pipelineMeta) { @@ -178,29 +173,27 @@ public String open() { PropsUi.setLook(wOutputField); wOutputField.setText(BaseMessages.getString(PKG, "YamlInputDialog.wOutputField.Label")); - FormLayout outputfieldgroupLayout = new FormLayout(); - outputfieldgroupLayout.marginWidth = 10; - outputfieldgroupLayout.marginHeight = 10; - wOutputField.setLayout(outputfieldgroupLayout); + FormLayout outPutFieldGroupLayout = new FormLayout(); + outPutFieldGroupLayout.marginWidth = 10; + outPutFieldGroupLayout.marginHeight = 10; + wOutputField.setLayout(outPutFieldGroupLayout); - // Is XML string defined in a Field + // Source is defined in a field Label wlXmlStreamField = new Label(wOutputField, SWT.RIGHT); wlXmlStreamField.setText(BaseMessages.getString(PKG, "YamlInputDialog.wlXmlStreamField.Label")); PropsUi.setLook(wlXmlStreamField); FormData fdlXMLStreamField = new FormData(); - fdlXMLStreamField.left = new FormAttachment(0, -margin); - fdlXMLStreamField.top = new FormAttachment(0, margin); - fdlXMLStreamField.right = new FormAttachment(middle, -margin); + fdlXMLStreamField.left = new FormAttachment(0, 0); + fdlXMLStreamField.right = new FormAttachment(middle, 0); wlXmlStreamField.setLayoutData(fdlXMLStreamField); wYAMLStreamField = new Button(wOutputField, SWT.CHECK); PropsUi.setLook(wYAMLStreamField); wYAMLStreamField.setToolTipText( BaseMessages.getString(PKG, "YamlInputDialog.wYAMLStreamField.Tooltip")); FormData fdYAMLStreamField = new FormData(); - fdYAMLStreamField.left = new FormAttachment(middle, -margin); - fdYAMLStreamField.top = new FormAttachment(wlXmlStreamField, 0, SWT.CENTER); + fdYAMLStreamField.left = new FormAttachment(wlXmlStreamField, margin); wYAMLStreamField.setLayoutData(fdYAMLStreamField); - SelectionAdapter lsyamlstream = + SelectionAdapter lsYamlStream = new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent arg0) { @@ -208,25 +201,25 @@ public void widgetSelected(SelectionEvent arg0) { input.setChanged(); } }; - wYAMLStreamField.addSelectionListener(lsyamlstream); + wYAMLStreamField.addSelectionListener(lsYamlStream); - // Is XML source is a file? + // Source is a filename wlYamlIsAFile = new Label(wOutputField, SWT.RIGHT); wlYamlIsAFile.setText(BaseMessages.getString(PKG, "YamlInputDialog.XMLIsAFile.Label")); PropsUi.setLook(wlYamlIsAFile); FormData fdlXMLIsAFile = new FormData(); - fdlXMLIsAFile.left = new FormAttachment(0, -margin); - fdlXMLIsAFile.top = new FormAttachment(wYAMLStreamField, margin); - fdlXMLIsAFile.right = new FormAttachment(middle, -margin); + fdlXMLIsAFile.left = new FormAttachment(0, 0); + fdlXMLIsAFile.top = new FormAttachment(wlXmlStreamField, margin); + fdlXMLIsAFile.right = new FormAttachment(middle, 0); wlYamlIsAFile.setLayoutData(fdlXMLIsAFile); wYAMLIsAFile = new Button(wOutputField, SWT.CHECK); PropsUi.setLook(wYAMLIsAFile); wYAMLIsAFile.setToolTipText(BaseMessages.getString(PKG, "YamlInputDialog.XMLIsAFile.Tooltip")); FormData fdYAMLIsAFile = new FormData(); - fdYAMLIsAFile.left = new FormAttachment(middle, -margin); - fdYAMLIsAFile.top = new FormAttachment(wlYamlIsAFile, 0, SWT.CENTER); + fdYAMLIsAFile.left = new FormAttachment(wlYamlIsAFile, margin); + fdYAMLIsAFile.top = new FormAttachment(wYAMLStreamField, margin); wYAMLIsAFile.setLayoutData(fdYAMLIsAFile); - SelectionAdapter lsyamlisafile = + SelectionAdapter lsYamlIsFile = new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent arg0) { @@ -234,16 +227,16 @@ public void widgetSelected(SelectionEvent arg0) { enableFileSettings(); } }; - wYAMLIsAFile.addSelectionListener(lsyamlisafile); + wYAMLIsAFile.addSelectionListener(lsYamlIsFile); - // If XML string defined in a Field + // Get source from a field wlYamlField = new Label(wOutputField, SWT.RIGHT); wlYamlField.setText(BaseMessages.getString(PKG, "YamlInputDialog.wlYamlField.Label")); PropsUi.setLook(wlYamlField); FormData fdlXMLField = new FormData(); - fdlXMLField.left = new FormAttachment(0, -margin); - fdlXMLField.top = new FormAttachment(wYAMLIsAFile, margin); - fdlXMLField.right = new FormAttachment(middle, -margin); + fdlXMLField.left = new FormAttachment(0, 0); + fdlXMLField.top = new FormAttachment(wlYamlIsAFile, margin); + fdlXMLField.right = new FormAttachment(middle, 0); wlYamlField.setLayoutData(fdlXMLField); wYAMLLField = new CCombo(wOutputField, SWT.BORDER | SWT.READ_ONLY); @@ -251,9 +244,9 @@ public void widgetSelected(SelectionEvent arg0) { PropsUi.setLook(wYAMLLField); wYAMLLField.addModifyListener(lsMod); FormData fdXMLField = new FormData(); - fdXMLField.left = new FormAttachment(middle, -margin); + fdXMLField.left = new FormAttachment(wlYamlField, margin); fdXMLField.top = new FormAttachment(wYAMLIsAFile, margin); - fdXMLField.right = new FormAttachment(100, -margin); + fdXMLField.right = new FormAttachment(100, 0); wYAMLLField.setLayoutData(fdXMLField); wYAMLLField.addFocusListener( new FocusListener() { @@ -376,40 +369,40 @@ public void focusGained(FocusEvent e) { fdbShowFiles.bottom = new FormAttachment(100, 0); wbShowFiles.setLayoutData(fdbShowFiles); - ColumnInfo[] colinfo = new ColumnInfo[4]; - colinfo[0] = + ColumnInfo[] colInfo = new ColumnInfo[4]; + colInfo[0] = new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.Files.Filename.Column"), ColumnInfo.COLUMN_TYPE_TEXT, false); - colinfo[1] = + colInfo[1] = new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.Files.Wildcard.Column"), ColumnInfo.COLUMN_TYPE_TEXT, false); - colinfo[0].setUsingVariables(true); - colinfo[1].setUsingVariables(true); - colinfo[1].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.Files.Wildcard.Tooltip")); - colinfo[2] = + colInfo[0].setUsingVariables(true); + colInfo[1].setUsingVariables(true); + colInfo[1].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.Files.Wildcard.Tooltip")); + colInfo[2] = new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.Required.Column"), ColumnInfo.COLUMN_TYPE_CCOMBO, NO_YES_DEC); - colinfo[2].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.Required.Tooltip")); - colinfo[3] = + colInfo[2].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.Required.Tooltip")); + colInfo[3] = new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.IncludeSubDirs.Column"), ColumnInfo.COLUMN_TYPE_CCOMBO, NO_YES_DEC); - colinfo[3].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.IncludeSubDirs.Tooltip")); + colInfo[3].setToolTip(BaseMessages.getString(PKG, "YamlInputDialog.IncludeSubDirs.Tooltip")); wFilenameList = new TableView( variables, wFileComp, SWT.FULL_SELECTION | SWT.SINGLE | SWT.BORDER, - colinfo, + colInfo, 2, lsMod, props); @@ -458,10 +451,10 @@ public void focusGained(FocusEvent e) { PropsUi.setLook(wXmlConf); wXmlConf.setText(BaseMessages.getString(PKG, "YamlInputDialog.wXmlConf.Label")); - FormLayout xmlConfgroupLayout = new FormLayout(); - xmlConfgroupLayout.marginWidth = 10; - xmlConfgroupLayout.marginHeight = 10; - wXmlConf.setLayout(xmlConfgroupLayout); + FormLayout xmlConfGroupLayout = new FormLayout(); + xmlConfGroupLayout.marginWidth = 10; + xmlConfGroupLayout.marginHeight = 10; + wXmlConf.setLayout(xmlConfGroupLayout); // Ignore Empty File wlIgnoreEmptyFile = new Label(wXmlConf, SWT.RIGHT); @@ -469,7 +462,6 @@ public void focusGained(FocusEvent e) { PropsUi.setLook(wlIgnoreEmptyFile); FormData fdlIgnoreEmptyFile = new FormData(); fdlIgnoreEmptyFile.left = new FormAttachment(0, 0); - fdlIgnoreEmptyFile.top = new FormAttachment(0, margin); fdlIgnoreEmptyFile.right = new FormAttachment(middle, -margin); wlIgnoreEmptyFile.setLayoutData(fdlIgnoreEmptyFile); wIgnoreEmptyFile = new Button(wXmlConf, SWT.CHECK); @@ -478,7 +470,6 @@ public void focusGained(FocusEvent e) { BaseMessages.getString(PKG, "YamlInputDialog.IgnoreEmptyFile.Tooltip")); FormData fdIgnoreEmptyFile = new FormData(); fdIgnoreEmptyFile.left = new FormAttachment(middle, 0); - fdIgnoreEmptyFile.top = new FormAttachment(wlIgnoreEmptyFile, 0, SWT.CENTER); wIgnoreEmptyFile.setLayoutData(fdIgnoreEmptyFile); wIgnoreEmptyFile.addSelectionListener(new ComponentSelectionListener(input)); @@ -538,17 +529,16 @@ public void focusGained(FocusEvent e) { wAdditionalFields.setText( BaseMessages.getString(PKG, "YamlInputDialog.wAdditionalFields.Label")); - FormLayout additionalFieldsgroupLayout = new FormLayout(); - additionalFieldsgroupLayout.marginWidth = 10; - additionalFieldsgroupLayout.marginHeight = 10; - wAdditionalFields.setLayout(additionalFieldsgroupLayout); + FormLayout additionalFieldsGroupLayout = new FormLayout(); + additionalFieldsGroupLayout.marginWidth = 10; + additionalFieldsGroupLayout.marginHeight = 10; + wAdditionalFields.setLayout(additionalFieldsGroupLayout); wlInclFilename = new Label(wAdditionalFields, SWT.RIGHT); wlInclFilename.setText(BaseMessages.getString(PKG, "YamlInputDialog.InclFilename.Label")); PropsUi.setLook(wlInclFilename); FormData fdlInclFilename = new FormData(); fdlInclFilename.left = new FormAttachment(0, 0); - fdlInclFilename.top = new FormAttachment(wXmlConf, 4 * margin); fdlInclFilename.right = new FormAttachment(middle, -margin); wlInclFilename.setLayoutData(fdlInclFilename); wInclFilename = new Button(wAdditionalFields, SWT.CHECK); @@ -557,7 +547,6 @@ public void focusGained(FocusEvent e) { BaseMessages.getString(PKG, "YamlInputDialog.InclFilename.Tooltip")); FormData fdInclFilename = new FormData(); fdInclFilename.left = new FormAttachment(middle, 0); - fdInclFilename.top = new FormAttachment(wlInclFilename, 0, SWT.CENTER); wInclFilename.setLayoutData(fdInclFilename); wInclFilename.addSelectionListener(new ComponentSelectionListener(input)); @@ -567,7 +556,6 @@ public void focusGained(FocusEvent e) { PropsUi.setLook(wlInclFilenameField); FormData fdlInclFilenameField = new FormData(); fdlInclFilenameField.left = new FormAttachment(wInclFilename, margin); - fdlInclFilenameField.top = new FormAttachment(wLimit, 4 * margin); wlInclFilenameField.setLayoutData(fdlInclFilenameField); wInclFilenameField = new TextVar(variables, wAdditionalFields, SWT.SINGLE | SWT.LEFT | SWT.BORDER); @@ -575,43 +563,42 @@ public void focusGained(FocusEvent e) { wInclFilenameField.addModifyListener(lsMod); FormData fdInclFilenameField = new FormData(); fdInclFilenameField.left = new FormAttachment(wlInclFilenameField, margin); - fdInclFilenameField.top = new FormAttachment(wLimit, 4 * margin); fdInclFilenameField.right = new FormAttachment(100, 0); wInclFilenameField.setLayoutData(fdInclFilenameField); - Label wlInclRownum = new Label(wAdditionalFields, SWT.RIGHT); - wlInclRownum.setText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownum.Label")); - PropsUi.setLook(wlInclRownum); - FormData fdlInclRownum = new FormData(); - fdlInclRownum.left = new FormAttachment(0, 0); - fdlInclRownum.top = new FormAttachment(wInclFilenameField, margin); - fdlInclRownum.right = new FormAttachment(middle, -margin); - wlInclRownum.setLayoutData(fdlInclRownum); - wInclRownum = new Button(wAdditionalFields, SWT.CHECK); - PropsUi.setLook(wInclRownum); - wInclRownum.setToolTipText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownum.Tooltip")); - FormData fdRownum = new FormData(); - fdRownum.left = new FormAttachment(middle, 0); - fdRownum.top = new FormAttachment(wlInclRownum, 0, SWT.CENTER); - wInclRownum.setLayoutData(fdRownum); - wInclRownum.addSelectionListener(new ComponentSelectionListener(input)); - - wlInclRownumField = new Label(wAdditionalFields, SWT.RIGHT); - wlInclRownumField.setText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownumField.Label")); - PropsUi.setLook(wlInclRownumField); - FormData fdlInclRownumField = new FormData(); - fdlInclRownumField.left = new FormAttachment(wInclRownum, margin); - fdlInclRownumField.top = new FormAttachment(wInclFilenameField, margin); - wlInclRownumField.setLayoutData(fdlInclRownumField); - wInclRownumField = + Label wlInclRowNum = new Label(wAdditionalFields, SWT.RIGHT); + wlInclRowNum.setText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownum.Label")); + PropsUi.setLook(wlInclRowNum); + FormData fdlInclRowNum = new FormData(); + fdlInclRowNum.left = new FormAttachment(0, 0); + fdlInclRowNum.top = new FormAttachment(wInclFilenameField, margin); + fdlInclRowNum.right = new FormAttachment(middle, -margin); + wlInclRowNum.setLayoutData(fdlInclRowNum); + wInclRowNum = new Button(wAdditionalFields, SWT.CHECK); + PropsUi.setLook(wInclRowNum); + wInclRowNum.setToolTipText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownum.Tooltip")); + FormData fdRowNum = new FormData(); + fdRowNum.left = new FormAttachment(middle, 0); + fdRowNum.top = new FormAttachment(wlInclRowNum, 0, SWT.CENTER); + wInclRowNum.setLayoutData(fdRowNum); + wInclRowNum.addSelectionListener(new ComponentSelectionListener(input)); + + wlInclRowNumField = new Label(wAdditionalFields, SWT.RIGHT); + wlInclRowNumField.setText(BaseMessages.getString(PKG, "YamlInputDialog.InclRownumField.Label")); + PropsUi.setLook(wlInclRowNumField); + FormData fdlInclRowNumField = new FormData(); + fdlInclRowNumField.left = new FormAttachment(wInclRowNum, margin); + fdlInclRowNumField.top = new FormAttachment(wInclFilenameField, margin); + wlInclRowNumField.setLayoutData(fdlInclRowNumField); + wInclRowNumField = new TextVar(variables, wAdditionalFields, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wInclRownumField); - wInclRownumField.addModifyListener(lsMod); - FormData fdInclRownumField = new FormData(); - fdInclRownumField.left = new FormAttachment(wlInclRownumField, margin); - fdInclRownumField.top = new FormAttachment(wInclFilenameField, margin); - fdInclRownumField.right = new FormAttachment(100, 0); - wInclRownumField.setLayoutData(fdInclRownumField); + PropsUi.setLook(wInclRowNumField); + wInclRowNumField.addModifyListener(lsMod); + FormData fdInclRowNumField = new FormData(); + fdInclRowNumField.left = new FormAttachment(wlInclRowNumField, margin); + fdInclRowNumField.top = new FormAttachment(wInclFilenameField, margin); + fdInclRowNumField.right = new FormAttachment(100, 0); + wInclRowNumField.setLayoutData(fdInclRowNumField); FormData fdAdditionalFields = new FormData(); fdAdditionalFields.left = new FormAttachment(0, margin); @@ -631,17 +618,16 @@ public void focusGained(FocusEvent e) { PropsUi.setLook(wAddFileResult); wAddFileResult.setText(BaseMessages.getString(PKG, "YamlInputDialog.wAddFileResult.Label")); - FormLayout addFileResultgroupLayout = new FormLayout(); - addFileResultgroupLayout.marginWidth = 10; - addFileResultgroupLayout.marginHeight = 10; - wAddFileResult.setLayout(addFileResultgroupLayout); + FormLayout addFileResultGroupLayout = new FormLayout(); + addFileResultGroupLayout.marginWidth = 10; + addFileResultGroupLayout.marginHeight = 10; + wAddFileResult.setLayout(addFileResultGroupLayout); wlAddResult = new Label(wAddFileResult, SWT.RIGHT); wlAddResult.setText(BaseMessages.getString(PKG, "YamlInputDialog.AddResult.Label")); PropsUi.setLook(wlAddResult); FormData fdlAddResult = new FormData(); fdlAddResult.left = new FormAttachment(0, 0); - fdlAddResult.top = new FormAttachment(wAdditionalFields, margin); fdlAddResult.right = new FormAttachment(middle, -margin); wlAddResult.setLayoutData(fdlAddResult); wAddResult = new Button(wAddFileResult, SWT.CHECK); @@ -649,7 +635,6 @@ public void focusGained(FocusEvent e) { wAddResult.setToolTipText(BaseMessages.getString(PKG, "YamlInputDialog.AddResult.Tooltip")); FormData fdAddResult = new FormData(); fdAddResult.left = new FormAttachment(middle, 0); - fdAddResult.top = new FormAttachment(wlAddResult, 0, SWT.CENTER); wAddResult.setLayoutData(fdAddResult); wAddResult.addSelectionListener(new ComponentSelectionListener(input)); @@ -700,7 +685,7 @@ public void focusGained(FocusEvent e) { final int FieldsRows = input.getInputFields().size(); - ColumnInfo[] colinf = + ColumnInfo[] colInf = new ColumnInfo[] { new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.FieldsTable.Name.Column"), @@ -742,15 +727,15 @@ public void focusGained(FocusEvent e) { new ColumnInfo( BaseMessages.getString(PKG, "YamlInputDialog.FieldsTable.TrimType.Column"), ColumnInfo.COLUMN_TYPE_CCOMBO, - YamlInputField.trimTypeDesc, + YamlInputField.TRIM_TYPE_DESC, true) }; - colinf[0].setUsingVariables(true); - colinf[0].setToolTip( + colInf[0].setUsingVariables(true); + colInf[0].setToolTip( BaseMessages.getString(PKG, "YamlInputDialog.FieldsTable.Name.Column.Tooltip")); - colinf[1].setUsingVariables(true); - colinf[1].setToolTip( + colInf[1].setUsingVariables(true); + colInf[1].setToolTip( BaseMessages.getString(PKG, "YamlInputDialog.FieldsTable.XPath.Column.Tooltip")); wFields = @@ -758,7 +743,7 @@ public void focusGained(FocusEvent e) { variables, wFieldsComp, SWT.FULL_SELECTION | SWT.MULTI, - colinf, + colInf, FieldsRows, lsMod, props); @@ -795,7 +780,7 @@ public void focusGained(FocusEvent e) { new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent arg0) { - wFilenameList.add(new String[] {wFilename.getText(), wFilemask.getText()}); + wFilenameList.add(wFilename.getText(), wFilemask.getText()); wFilename.setText(""); wFilemask.setText(""); wFilenameList.removeEmptyRows(); @@ -841,9 +826,9 @@ public void widgetSelected(SelectionEvent arg0) { @Override public void widgetSelected(SelectionEvent e) { try { - YamlInputMeta tfii = new YamlInputMeta(); - getInfo(tfii); - FileInputList fileInputList = tfii.getFiles(variables); + YamlInputMeta im = new YamlInputMeta(); + getInfo(im); + FileInputList fileInputList = im.getFiles(variables); String[] files = fileInputList.getFileStrings(); if (files != null && files.length > 0) { EnterSelectionDialog esd = @@ -884,11 +869,11 @@ public void widgetSelected(SelectionEvent e) { }); // Enable/disable the right fields to allow a row number to be added to each row... - wInclRownum.addSelectionListener( + wInclRowNum.addSelectionListener( new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent e) { - setIncludeRownum(); + setIncludeRowNum(); } }); @@ -896,29 +881,24 @@ public void widgetSelected(SelectionEvent e) { wFilename.addModifyListener(e -> wFilename.setToolTipText(wFilename.getText())); // Listen to the Browse... button - if (wbbFilename != null) { - // Listen to the browse button next to the file name - // - wbbFilename.addListener( - SWT.Selection, - e -> - BaseDialog.presentFileDialog( - shell, - wFilename, - variables, - new String[] {"*.yaml;*.YAML;*.yml;*.YML", "*"}, - new String[] { - "Yaml files", BaseMessages.getString(PKG, "System.FileType.AllFiles") - }, - true)); - } - + wbbFilename.addListener( + SWT.Selection, + e -> + BaseDialog.presentFileDialog( + shell, + wFilename, + variables, + new String[] {"*.yaml;*.YAML;*.yml;*.YML", "*"}, + new String[] { + "Yaml files", BaseMessages.getString(PKG, "System.FileType.AllFiles") + }, + true)); wTabFolder.setSelection(0); getData(input); activateStreamField(); setIncludeFilename(); - setIncludeRownum(); + setIncludeRowNum(); input.setChanged(changed); wFields.optWidth(true); @@ -931,7 +911,6 @@ public void widgetSelected(SelectionEvent e) { private void setXMLStreamField() { try { - wYAMLLField.removeAll(); IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); @@ -1049,9 +1028,9 @@ public void setIncludeFilename() { wInclFilenameField.setEnabled(wInclFilename.getSelection()); } - public void setIncludeRownum() { - wlInclRownumField.setEnabled(wInclRownum.getSelection()); - wInclRownumField.setEnabled(wInclRownum.getSelection()); + public void setIncludeRowNum() { + wlInclRowNumField.setEnabled(wInclRowNum.getSelection()); + wInclRowNumField.setEnabled(wInclRowNum.getSelection()); } /** @@ -1071,7 +1050,7 @@ public void getData(YamlInputMeta in) { wFilenameList.optimizeTableView(); wInclFilename.setSelection(in.isIncludeFilename()); - wInclRownum.setSelection(in.isIncludeRowNumber()); + wInclRowNum.setSelection(in.isIncludeRowNumber()); wAddResult.setSelection(in.isAddingResultFile()); wIgnoreEmptyFile.setSelection(in.isIgnoringEmptyFile()); wDoNotFailIfNoFile.setSelection(in.isDoNotFailIfNoFile()); @@ -1079,7 +1058,7 @@ public void getData(YamlInputMeta in) { wYAMLIsAFile.setSelection(in.isSourceFile()); wYAMLLField.setText(Const.NVL(in.getYamlField(), "")); wInclFilenameField.setText(Const.NVL(in.getFilenameField(), "")); - wInclRownumField.setText(Const.NVL(in.getRowNumberField(), "")); + wInclRowNumField.setText(Const.NVL(in.getRowNumberField(), "")); wLimit.setText("" + in.getRowLimit()); if (isDebug()) { @@ -1089,8 +1068,8 @@ public void getData(YamlInputMeta in) { YamlInputField field = in.getInputFields().get(i); TableItem item = wFields.table.getItem(i); String length = "" + field.getLength(); - String prec = "" + field.getPrecision(); - String decim = field.getDecimalSymbol(); + String precision = "" + field.getPrecision(); + String decimalSymbol = field.getDecimalSymbol(); item.setText(1, Const.NVL(field.getName(), "")); item.setText(2, Const.NVL(field.getPath(), "")); @@ -1099,12 +1078,12 @@ public void getData(YamlInputMeta in) { if (!"-1".equals(length)) { item.setText(5, length); } - if (!"-1".equals(prec)) { - item.setText(6, prec); + if (!"-1".equals(precision)) { + item.setText(6, precision); } item.setText(7, Const.NVL(field.getCurrencySymbol(), "")); - if (decim != null) { - item.setText(8, decim); + if (decimalSymbol != null) { + item.setText(8, decimalSymbol); } item.setText(9, Const.NVL(field.getGroupSymbol(), "")); item.setText(10, Const.NVL(field.getTrimTypeDesc(), "")); @@ -1132,15 +1111,16 @@ private void ok() { } private void getInfo(YamlInputMeta in) { - transformName = wTransformName.getText(); // return value + // return value + transformName = wTransformName.getText(); // copy info to TextFileInputMeta class (input) in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); in.setFilenameField(wInclFilenameField.getText()); - in.setRowNumberField(wInclRownumField.getText()); + in.setRowNumberField(wInclRowNumField.getText()); in.setAddingResultFile(wAddResult.getSelection()); in.setIncludeFilename(wInclFilename.getSelection()); - in.setIncludeRowNumber(wInclRownum.getSelection()); + in.setIncludeRowNumber(wInclRowNum.getSelection()); in.setIgnoringEmptyFile(wIgnoreEmptyFile.getSelection()); in.setDoNotFailIfNoFile(wDoNotFailIfNoFile.getSelection()); diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java index c54827563ba..0d694120cbc 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java @@ -39,15 +39,12 @@ public class YamlInputField implements Cloneable { public static final int TYPE_TRIM_RIGHT = 2; public static final int TYPE_TRIM_BOTH = 3; - public static final String[] trimTypeCode = {"none", "left", "right", "both"}; - - public static final String[] trimTypeDesc = { + protected static final String[] TRIM_TYPE_DESC = { BaseMessages.getString(PKG, "YamlInputField.TrimType.None"), BaseMessages.getString(PKG, "YamlInputField.TrimType.Left"), BaseMessages.getString(PKG, "YamlInputField.TrimType.Right"), BaseMessages.getString(PKG, "YamlInputField.TrimType.Both") }; - public static final String CONST_SPACES = " "; @HopMetadataProperty(key = "name") private String name; @@ -100,7 +97,7 @@ public YamlInputField() { public YamlInputField(YamlInputField f) { this(); - this.name = name; + this.name = f.name; this.path = f.path; this.type = f.type; this.length = f.length; diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java index 0c02b3923bf..5ff18268158 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java @@ -245,7 +245,8 @@ public FileInputList getFiles(IVariables variables) { inputFile.setFileMask(file.getFileMask()); inputFile.setIncludeSubFolders(file.isIncludingSubFolders()); inputFile.setFileRequired(file.isFileRequired()); - inputFile.setExcludeFileMask(null); // Not provided in this transform + // Not provided in this transform + inputFile.setExcludeFileMask(null); inputFiles.add(inputFile); } return FileInputList.createFileList(variables, inputFiles, null); diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java index 310e4c9c3e2..d0acefd6dab 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java @@ -17,18 +17,18 @@ package org.apache.hop.pipeline.transforms.yamlinput; -import java.io.InputStream; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; -import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopPluginException; import org.apache.hop.core.io.CountingInputStream; import org.apache.hop.core.row.IRowMeta; @@ -37,7 +37,6 @@ import org.apache.hop.core.row.value.ValueMetaFactory; import org.apache.hop.core.row.value.ValueMetaNone; import org.apache.hop.core.util.Utils; -import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.yaml.snakeyaml.Yaml; @@ -46,36 +45,31 @@ * streams. */ public class YamlReader { - private static final String DEFAULT_LIST_VALUE_NAME = "Value"; - private String filename; private String string; - private FileObject file; + @Getter private FileObject file; // document // Store all documents available private List documents; // Store current document - private Object document; + @Getter private Object document; // Store document iterator - private Iterator documenti; + private Iterator documentIt; // object current inside a document // In case we use a list private Object dataList; // Store object iterator - private Iterator dataListi; - + private Iterator dataListIt; private boolean useMap; - - private Yaml yaml; + @Getter private Yaml yaml; /** Bytes read from the last loaded file (for data volume tracking). */ - private long bytesReadFromFile; + @Getter private long bytesReadFromFile; public YamlReader() { - this.filename = null; this.string = null; this.file = null; this.documents = new ArrayList<>(); @@ -86,44 +80,17 @@ public YamlReader() { public void loadFile(FileObject file) throws Exception { this.file = file; - this.filename = HopVfs.getFilename(file); - - InputStream is = null; - try { - is = HopVfs.getInputStream(getFile()); - CountingInputStream countingStream = new CountingInputStream(is); - is = countingStream; + try (CountingInputStream is = new CountingInputStream(HopVfs.getInputStream(getFile()))) { for (Object data : getYaml().loadAll(is)) { documents.add(data); this.useMap = (data instanceof Map); } - bytesReadFromFile = countingStream.getCount(); - this.documenti = documents.iterator(); - - } finally { - if (is != null) { - is.close(); - } + bytesReadFromFile = is.getCount(); + this.documentIt = documents.iterator(); } } - /** Returns bytes read from the last file loaded via {@link #loadFile(FileObject)}. */ - public long getBytesReadFromFile() { - return bytesReadFromFile; - } - - public void loadFile(String filename, IVariables variables) throws Exception { - this.filename = filename; - this.file = HopVfs.getFileObject(filename, variables); - - loadFile(this.file); - } - - private Yaml getYaml() { - return this.yaml; - } - public void loadString(String string) { this.string = string; bytesReadFromFile = 0; @@ -131,298 +98,152 @@ public void loadString(String string) { documents.add(data); this.useMap = (data instanceof Map); } - this.documenti = documents.iterator(); + this.documentIt = documents.iterator(); } public boolean isMapUsed() { return this.useMap; } - public Object[] getRow(IRowMeta rowMeta) throws HopException { - - Object[] retval = null; - - if (getDocument() != null) { - if (isMapUsed()) { - Map map = (Map) getDocument(); - retval = new Object[rowMeta.size()]; - for (int i = 0; i < rowMeta.size(); i++) { - IValueMeta valueMeta = rowMeta.getValueMeta(i); - Object o = null; - if (Utils.isEmpty(valueMeta.getName())) { - o = getDocument().toString(); - } else { - o = map.get(valueMeta.getName()); - } - retval[i] = getValue(o, valueMeta); - } - - // We have done with this document - finishDocument(); - } else { - if (dataList != null) { - - List list = (List) getDocument(); - if (list.size() == 1) { - Iterator it = list.iterator(); - Object value = it.next(); - Map map = (Map) value; - retval = new Object[rowMeta.size()]; - for (int i = 0; i < rowMeta.size(); i++) { - IValueMeta valueMeta = rowMeta.getValueMeta(i); - Object o = null; - if (Utils.isEmpty(valueMeta.getName())) { - o = getDocument().toString(); - } else { - o = map.get(valueMeta.getName()); - } - retval[i] = getValue(o, valueMeta); - } - } else { - - IValueMeta valueMeta = rowMeta.getValueMeta(0); - retval = new Object[1]; - - retval[0] = getValue(dataList, valueMeta); - } - dataList = null; - } else { - // We are using List - if (dataListi.hasNext()) { - dataList = dataListi.next(); - } else { - // We have done with this document - finishDocument(); - } - } - } - } else { - // See if we have another document + /** get row */ + public Object[] getRow(IRowMeta rowMeta) { + if (document == null) { getNextDocument(); } - if (retval == null && !isfinishedDocument()) { + if (document == null) { + return new Object[0]; + } + + Object[] row = fetchRow(rowMeta); + if (row.length == 0 && !isFinishedDocument()) { return getRow(rowMeta); } - return retval; + return row; } - private Object getValue(Object value, IValueMeta valueMeta) { + private Object[] fetchRow(IRowMeta rowMeta) { + if (isMapUsed()) { + return processMapRow(rowMeta, (Map) document); + } - if (value == null) { - return null; + return processListRow(rowMeta); + } + + private Object[] handleCurrentListValue(IRowMeta rowMeta) { + List list = (List) document; + + if (list.size() == 1 && list.getFirst() instanceof Map map) { + return processMapRow(rowMeta, map); } - Object o = null; - if (value instanceof List) { - value = getYaml().dump(value); + IValueMeta valueMeta = rowMeta.getValueMeta(0); + return new Object[] {getValue(dataList, valueMeta)}; + } + + private Object[] processListRow(IRowMeta rowMeta) { + if (dataList == null) { + return advanceListIterator(); } - switch (valueMeta.getType()) { - case IValueMeta.TYPE_INTEGER: - if (value instanceof Integer integerValue) { - o = Long.valueOf(integerValue); - } else if (value instanceof BigInteger bigIntegerValue) { - o = bigIntegerValue.longValue(); - } else if (value instanceof Long longValue) { - o = Long.valueOf(longValue); - } else { - o = Long.valueOf(value.toString()); - } - break; - case IValueMeta.TYPE_NUMBER: - if (value instanceof Integer integerValue) { - o = Double.valueOf(integerValue); - } else if (value instanceof BigInteger bigIntegerValue) { - o = bigIntegerValue.doubleValue(); - } else if (value instanceof Long longValue) { - o = Double.valueOf(longValue); - } else if (value instanceof Double) { - o = value; - } else { - o = Double.valueOf((String) value); - } - break; - case IValueMeta.TYPE_BIGNUMBER: - if (value instanceof Integer integerValue) { - o = new BigDecimal(integerValue); - } else if (value instanceof BigInteger bigIntegerValue) { - o = new BigDecimal(bigIntegerValue); - } else if (value instanceof Long longValue) { - o = new BigDecimal(longValue); - } else if (value instanceof Double doubleValue) { - o = BigDecimal.valueOf(doubleValue); - } - break; - case IValueMeta.TYPE_BOOLEAN: - o = value; - break; - case IValueMeta.TYPE_DATE: - o = value; - break; - case IValueMeta.TYPE_BINARY: - o = value; - break; - default: - String s = setMap(value); - - // Do trimming - switch (valueMeta.getTrimType()) { - case YamlInputField.TYPE_TRIM_LEFT: - s = Const.ltrim(s); - break; - case YamlInputField.TYPE_TRIM_RIGHT: - s = Const.rtrim(s); - break; - case YamlInputField.TYPE_TRIM_BOTH: - s = Const.trim(s); - break; - default: - break; - } - o = s; + Object[] row = handleCurrentListValue(rowMeta); + dataList = null; + return row; + } - break; + private Object[] processMapRow(IRowMeta rowMeta, Map map) { + Object[] row = new Object[rowMeta.size()]; + + for (int i = 0; i < rowMeta.size(); i++) { + IValueMeta valueMeta = rowMeta.getValueMeta(i); + + Object value = + Utils.isEmpty(valueMeta.getName()) ? document.toString() : map.get(valueMeta.getName()); + + row[i] = getValue(value, valueMeta); + } + + finishDocument(); + return row; + } + + private Object[] advanceListIterator() { + if (!dataListIt.hasNext()) { + finishDocument(); + return new Object[0]; } - return o; + + dataList = dataListIt.next(); + return new Object[0]; } private void getNextDocument() { // See if we have another document - if (this.documenti.hasNext()) { + if (this.documentIt.hasNext()) { // We have another document - this.document = this.documenti.next(); + this.document = this.documentIt.next(); if (!isMapUsed()) { + @SuppressWarnings("unchecked") List list = (List) getDocument(); - dataListi = list.iterator(); + dataListIt = list.iterator(); } } } private String setMap(Object value) { - String result = value.toString(); + StringBuilder result = new StringBuilder(value.toString()); if (value instanceof Map) { - + @SuppressWarnings("unchecked") Map map = (Map) value; - Iterator it = map.entrySet().iterator(); + Iterator> it = map.entrySet().iterator(); int nr = 0; while (it.hasNext()) { - Map.Entry pairs = (Map.Entry) it.next(); + Map.Entry pairs = it.next(); String res = pairs.getKey().toString() + ": " + setMap(pairs.getValue()); if (nr == 0) { - result = "{" + res; + result = new StringBuilder("{" + res); } else { - result += "," + res; + result.append(",").append(res); } nr++; } if (nr > 0) { - result += "}"; - } - } - return result; - } - - public RowMeta getFields() { - RowMeta rowMeta = new RowMeta(); - - Iterator ito = documents.iterator(); - while (ito.hasNext()) { - Object data = ito.next(); - if (data instanceof Map) { - // First check if we deals with a map - - Map map = (Map) data; - Iterator it = map.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry pairs = (Map.Entry) it.next(); - String valueName = pairs.getKey().toString(); - IValueMeta valueMeta; - try { - valueMeta = ValueMetaFactory.createValueMeta(valueName, getType(pairs.getValue())); - } catch (HopPluginException e) { - valueMeta = new ValueMetaNone(valueName); - } - rowMeta.addValueMeta(valueMeta); - } - } else if (data instanceof List) { - - rowMeta = new RowMeta(); - // Maybe we deals with List - List list = (List) data; - Iterator it = list.iterator(); - Object value = it.next(); - - if (list.size() == 1) { - Map map = (Map) value; - Iterator its = map.entrySet().iterator(); - while (its.hasNext()) { - Map.Entry pairs = (Map.Entry) its.next(); - String valueName = pairs.getKey().toString(); - IValueMeta valueMeta; - try { - valueMeta = ValueMetaFactory.createValueMeta(valueName, getType(pairs.getValue())); - } catch (HopPluginException e) { - valueMeta = new ValueMetaNone(valueName); - } - rowMeta.addValueMeta(valueMeta); - } - } else { - IValueMeta valueMeta; - try { - valueMeta = ValueMetaFactory.createValueMeta(DEFAULT_LIST_VALUE_NAME, getType(value)); - } catch (HopPluginException e) { - valueMeta = new ValueMetaNone(DEFAULT_LIST_VALUE_NAME); - } - rowMeta.addValueMeta(valueMeta); - } + result.append("}"); } } - - return rowMeta; + return result.toString(); } private int getType(Object value) { - - if (value instanceof Integer) { - return IValueMeta.TYPE_INTEGER; - } - if (value instanceof Double) { - return IValueMeta.TYPE_NUMBER; - } else if (value instanceof Long) { - return IValueMeta.TYPE_INTEGER; - } else if (value instanceof Date) { - return IValueMeta.TYPE_DATE; - } else if (value instanceof java.sql.Date) { - return IValueMeta.TYPE_DATE; - } else if (value instanceof Timestamp) { - return IValueMeta.TYPE_DATE; - } else if (value instanceof Boolean) { - return IValueMeta.TYPE_BOOLEAN; - } else if (value instanceof BigInteger) { - return IValueMeta.TYPE_BIGNUMBER; - } else if (value instanceof BigDecimal) { - return IValueMeta.TYPE_BIGNUMBER; - } else if (value instanceof Byte) { - return IValueMeta.TYPE_BINARY; + if (value == null) { + return IValueMeta.TYPE_STRING; } - return IValueMeta.TYPE_STRING; - } - private Object getDocument() { - return this.document; + return switch (value) { + case Integer ignored -> IValueMeta.TYPE_INTEGER; + case Long ignored -> IValueMeta.TYPE_INTEGER; + case Double ignored -> IValueMeta.TYPE_NUMBER; + case Boolean ignored -> IValueMeta.TYPE_BOOLEAN; + case BigInteger ignored -> IValueMeta.TYPE_BIGNUMBER; + case BigDecimal ignored -> IValueMeta.TYPE_BIGNUMBER; + case Byte ignored -> IValueMeta.TYPE_BINARY; + case Timestamp ignored -> IValueMeta.TYPE_DATE; + case java.sql.Date ignored -> IValueMeta.TYPE_DATE; + case java.util.Date ignored -> IValueMeta.TYPE_DATE; + default -> IValueMeta.TYPE_STRING; + }; } private void finishDocument() { this.document = null; } - private boolean isfinishedDocument() { + private boolean isFinishedDocument() { return (this.document == null); } - public void close() throws Exception { + public void close() throws IOException { if (file != null) { file.close(); } @@ -430,11 +251,141 @@ public void close() throws Exception { this.yaml = null; } - public FileObject getFile() { - return this.file; - } - public String getStringValue() { return this.string; } + + /** get value */ + private Object getValue(Object value, IValueMeta valueMeta) { + if (value == null) { + return null; + } + + Object normalizedValue = normalizeList(value); + + return switch (valueMeta.getType()) { + case IValueMeta.TYPE_INTEGER -> convertToInteger(normalizedValue); + case IValueMeta.TYPE_NUMBER -> convertToDouble(normalizedValue); + case IValueMeta.TYPE_BIGNUMBER -> convertToBigDecimal(normalizedValue); + case IValueMeta.TYPE_BOOLEAN, IValueMeta.TYPE_DATE, IValueMeta.TYPE_BINARY -> normalizedValue; + default -> convertToString(normalizedValue, valueMeta); + }; + } + + private Object normalizeList(Object value) { + if (value instanceof List) { + String dumped = getYaml().dump(value); + if (StringUtils.isEmpty(dumped)) { + return dumped; + } + + return stripTrailingLineBreaks(dumped); + } + return value; + } + + /** SnakeYAML {@code dump} ends the stream with a line break; strip it for cell values. */ + private static String stripTrailingLineBreaks(String s) { + int end = s.length(); + while (end > 0) { + char c = s.charAt(end - 1); + if (c != '\n' && c != '\r') { + break; + } + end--; + } + return s.substring(0, end); + } + + /** object -> long */ + private Long convertToInteger(Object value) { + return switch (value) { + case Integer i -> i.longValue(); + case BigInteger bi -> bi.longValue(); + case Long l -> l; + default -> Long.valueOf(value.toString()); + }; + } + + /** object -> double */ + private Double convertToDouble(Object value) { + return switch (value) { + case Integer i -> i.doubleValue(); + case BigInteger bi -> bi.doubleValue(); + case Long l -> l.doubleValue(); + case Double d -> d; + default -> Double.valueOf(value.toString()); + }; + } + + private BigDecimal convertToBigDecimal(Object value) { + return switch (value) { + case Integer i -> new BigDecimal(i); + case BigInteger bi -> new BigDecimal(bi); + case Long l -> BigDecimal.valueOf(l); + case Double d -> BigDecimal.valueOf(d); + default -> null; + }; + } + + private String convertToString(Object value, IValueMeta valueMeta) { + String s = setMap(value); + return applyTrim(s, valueMeta); + } + + private String applyTrim(String s, IValueMeta valueMeta) { + return switch (valueMeta.getTrimType()) { + case YamlInputField.TYPE_TRIM_LEFT -> Const.ltrim(s); + case YamlInputField.TYPE_TRIM_RIGHT -> Const.rtrim(s); + case YamlInputField.TYPE_TRIM_BOTH -> Const.trim(s); + default -> s; + }; + } + + /** get fields. */ + public RowMeta getFields() { + RowMeta rowMeta = new RowMeta(); + + for (Object data : documents) { + if (data instanceof Map map) { + appendFromMap(rowMeta, map); + } else if (data instanceof List list) { + appendFromList(rowMeta, list); + } + } + + return rowMeta; + } + + private void appendFromList(RowMeta rowMeta, List list) { + if (list.isEmpty()) { + return; + } + + Object first = list.getFirst(); + if (list.size() == 1 && first instanceof Map map) { + appendFromMap(rowMeta, map); + } else { + addField(rowMeta, DEFAULT_LIST_VALUE_NAME, first); + } + } + + private void appendFromMap(RowMeta rowMeta, Map map) { + for (Map.Entry entry : map.entrySet()) { + addField(rowMeta, entry.getKey(), entry.getValue()); + } + } + + private void addField(RowMeta rowMeta, Object key, Object value) { + String fieldName = String.valueOf(key); + + IValueMeta valueMeta; + try { + valueMeta = ValueMetaFactory.createValueMeta(fieldName, getType(value)); + } catch (HopPluginException e) { + valueMeta = new ValueMetaNone(fieldName); + } + + rowMeta.addValueMeta(valueMeta); + } } diff --git a/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties b/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties index 7f90292da5b..94f75a32661 100644 --- a/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties +++ b/plugins/transforms/yamlinput/src/main/resources/org/apache/hop/pipeline/transforms/yamlinput/messages/messages_en_US.properties @@ -70,7 +70,7 @@ YamlInputDialog.FilenameAdd.Tooltip=Add this entry to the list of files & direct YamlInputDialog.FilenameBrowse.Button=&Browse YamlInputDialog.FilenameEdit.Button=&Edit YamlInputDialog.FilenameEdit.Tooltip=Edit the selected file and remove from the list. -YamlInputDialog.FilenameList.Label=Selected files\: +YamlInputDialog.FilenameList.Label=Selected files YamlInputDialog.FilenameRemove.Button=&Delete YamlInputDialog.FilenameRemove.Tooltip=Delete the selected entries from the list of files. YamlInputDialog.Files.Filename.Column=File/Directory diff --git a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDataTests.java b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDataTests.java new file mode 100644 index 00000000000..35a407e2aca --- /dev/null +++ b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDataTests.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.yamlinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +/** Unit test for {@link YamlInputData} */ +class YamlInputDataTests { + + @Test + void testDefaultConstructor() { + YamlInputData data = new YamlInputData(); + assertNotNull(data); + + assertEquals(0, data.fileIndex); + assertEquals(-1, data.indexOfYamlField); + assertEquals(-1, data.nrInputFields); + assertEquals(0, data.totalPreviousFields); + assertEquals(0, data.totalOutFields); + assertEquals(0, data.totalOutStreamFields); + + assertEquals(0L, data.rowIndex); + + assertNull(data.file); + assertNull(data.readRow); + assertNull(data.yaml); + + assertNull(data.files); + assertNull(data.outputRowMeta); + assertNull(data.rowMeta); + } +} diff --git a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputFieldTests.java b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputFieldTests.java new file mode 100644 index 00000000000..9e96de5fab0 --- /dev/null +++ b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputFieldTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.yamlinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; + +import org.apache.hop.core.row.IValueMeta; +import org.junit.jupiter.api.Test; + +/** Unit test for {@link YamlInputField} */ +class YamlInputFieldTests { + + @Test + void testDefaultConstructor() { + YamlInputField field = new YamlInputField(); + + assertEquals("", field.getName()); + assertEquals("", field.getPath()); + assertEquals(-1, field.getLength()); + assertEquals(-1, field.getPrecision()); + assertEquals("", field.getFormat()); + assertEquals("", field.getGroupSymbol()); + assertEquals("", field.getDecimalSymbol()); + assertEquals("", field.getCurrencySymbol()); + + assertEquals(IValueMeta.TYPE_STRING, field.getType()); + assertEquals(YamlInputField.TYPE_TRIM_NONE, field.getTrimType()); + } + + @Test + void testConstructorWithName() { + YamlInputField field = new YamlInputField("testField"); + + assertEquals("testField", field.getName()); + } + + @Test + void testCopyConstructor() { + YamlInputField original = new YamlInputField("name"); + original.setPath("$.data.name"); + original.setType(IValueMeta.TYPE_INTEGER); + original.setLength(10); + original.setFormat("###"); + original.setTrimType(YamlInputField.TYPE_TRIM_BOTH); + original.setPrecision(2); + original.setCurrencySymbol("$"); + original.setDecimalSymbol("."); + original.setGroupSymbol(","); + + YamlInputField copy = new YamlInputField(original); + + assertEquals(original.getName(), copy.getName()); + assertEquals(original.getPath(), copy.getPath()); + assertEquals(original.getType(), copy.getType()); + assertEquals(original.getLength(), copy.getLength()); + assertEquals(original.getFormat(), copy.getFormat()); + assertEquals(original.getTrimType(), copy.getTrimType()); + assertEquals(original.getPrecision(), copy.getPrecision()); + assertEquals(original.getCurrencySymbol(), copy.getCurrencySymbol()); + assertEquals(original.getDecimalSymbol(), copy.getDecimalSymbol()); + assertEquals(original.getGroupSymbol(), copy.getGroupSymbol()); + } + + @Test + void testClone() { + YamlInputField original = new YamlInputField("cloneField"); + original.setPath("$.path"); + + YamlInputField cloned = (YamlInputField) original.clone(); + + assertNotSame(original, cloned); + assertEquals(original.getName(), cloned.getName()); + assertEquals(original.getPath(), cloned.getPath()); + } + + @Test + void testTrimTypeDesc() { + YamlInputField field = new YamlInputField(); + field.setTrimType(YamlInputField.TYPE_TRIM_BOTH); + + String desc = field.getTrimTypeDesc(); + assertNotNull(desc); + } + + @Test + void testStaticTrimTypeMethods() { + String desc = YamlInputField.getTrimTypeDesc(YamlInputField.TYPE_TRIM_LEFT); + assertNotNull(desc); + + int type = YamlInputField.getTrimTypeByDesc(desc); + assertEquals(YamlInputField.TYPE_TRIM_LEFT, type); + } + + @Test + void testGetTypeDesc() { + YamlInputField field = new YamlInputField(); + field.setType(IValueMeta.TYPE_INTEGER); + + String desc = field.getTypeDesc(); + assertNotNull(desc); + } +} diff --git a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java index 42360c42367..d71a68e1fd9 100644 --- a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java +++ b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java @@ -17,10 +17,22 @@ package org.apache.hop.pipeline.transforms.yamlinput; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.fileinput.FileInputList; import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.row.value.ValueMetaDate; import org.apache.hop.core.row.value.ValueMetaInteger; import org.apache.hop.core.row.value.ValueMetaJson; @@ -28,11 +40,18 @@ import org.apache.hop.core.row.value.ValueMetaPlugin; import org.apache.hop.core.row.value.ValueMetaPluginType; import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.w3c.dom.Node; +/** Unit test for {@link YamlInputMeta} */ class YamlInputMetaTest { + @BeforeEach void beforeEach() throws Exception { PluginRegistry registry = PluginRegistry.getInstance(); @@ -80,6 +99,10 @@ void testSerializationRoundTrip() throws Exception { assertTrue(file.isFileRequired()); assertTrue(file.isIncludingSubFolders()); + testSerializationRoundTrip_1(meta); + } + + private void testSerializationRoundTrip_1(YamlInputMeta meta) { assertEquals(3, meta.getInputFields().size()); YamlInputField field = meta.getInputFields().getFirst(); assertEquals("field1", field.getName()); @@ -103,4 +126,199 @@ void testSerializationRoundTrip() throws Exception { assertEquals(2, field.getPrecision()); assertEquals(IValueMeta.TRIM_TYPE_BOTH, field.getTrimType()); } + + @Test + void testDefaultConstructor() { + YamlInputMeta meta = new YamlInputMeta(); + + assertNotNull(meta.getYamlFiles()); + assertNotNull(meta.getInputFields()); + + assertTrue(meta.getYamlFiles().isEmpty()); + assertTrue(meta.getInputFields().isEmpty()); + + assertEquals("", meta.getFilenameField()); + assertEquals("", meta.getRowNumberField()); + assertEquals("", meta.getYamlField()); + assertEquals(0, meta.getRowLimit()); + + assertTrue(meta.isDoNotFailIfNoFile()); + } + + @Test + void testCloneDeepCopy() { + YamlInputMeta meta = new YamlInputMeta(); + + YamlInputMeta.YamlFile file = new YamlInputMeta.YamlFile(); + file.setFilename("test.yaml"); + meta.getYamlFiles().add(file); + + YamlInputField field = new YamlInputField("name"); + meta.getInputFields().add(field); + + YamlInputMeta clone = meta.clone(); + + assertNotSame(meta, clone); + + assertNotSame(meta.getYamlFiles(), clone.getYamlFiles()); + assertNotSame(meta.getInputFields(), clone.getInputFields()); + + assertNotSame(meta.getYamlFiles().getFirst(), clone.getYamlFiles().getFirst()); + assertNotSame(meta.getInputFields().getFirst(), clone.getInputFields().getFirst()); + } + + @Test + void testGetFields_basic() throws Exception { + YamlInputMeta meta = new YamlInputMeta(); + + YamlInputField field = new YamlInputField("id"); + field.setType(IValueMeta.TYPE_INTEGER); + field.setLength(10); + field.setPrecision(0); + + meta.getInputFields().add(field); + + IRowMeta rowMeta = new RowMeta(); + Variables vars = new Variables(); + + meta.getFields(rowMeta, "testTransform", null, null, vars, null); + + assertEquals(1, rowMeta.size()); + + IValueMeta v = rowMeta.getValueMeta(0); + assertEquals("id", v.getName()); + assertEquals(IValueMeta.TYPE_INTEGER, v.getType()); + assertEquals(10, v.getLength()); + } + + @Test + void testGetFields_withExtraFields() throws Exception { + YamlInputMeta meta = new YamlInputMeta(); + + meta.setIncludeFilename(true); + meta.setFilenameField("filename"); + + meta.setIncludeRowNumber(true); + meta.setRowNumberField("rowNum"); + + IRowMeta rowMeta = new RowMeta(); + Variables vars = new Variables(); + + meta.getFields(rowMeta, "t", null, null, vars, null); + + assertEquals(2, rowMeta.size()); + assertEquals("filename", rowMeta.getValueMeta(0).getName()); + assertEquals("rowNum", rowMeta.getValueMeta(1).getName()); + } + + @Test + void testGetFiles(@TempDir Path tempDir) throws IOException { + YamlInputMeta meta = new YamlInputMeta(); + + Path tempFile = tempDir.resolve("test.yaml"); + Files.createFile(tempFile); + + YamlInputMeta.YamlFile file = new YamlInputMeta.YamlFile(); + file.setFilename(tempFile.toString()); + file.setFileMask(".*yaml"); + + meta.getYamlFiles().add(file); + + Variables vars = new Variables(); + FileInputList list = meta.getFiles(vars); + + assertNotNull(list); + assertFalse(list.getFiles().isEmpty()); + } + + @Test + void testCheck_noInput() { + YamlInputMeta meta = new YamlInputMeta(); + List remarks = new ArrayList<>(); + + meta.check( + remarks, + null, + new TransformMeta(), + null, + new String[] {}, + null, + null, + new Variables(), + null); + + assertFalse(remarks.isEmpty()); + } + + @Test + void testCheck_noFields() { + YamlInputMeta meta = new YamlInputMeta(); + List remarks = new ArrayList<>(); + + meta.check( + remarks, + null, + new TransformMeta(), + null, + new String[] {"input"}, + null, + null, + new Variables(), + null); + + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR)); + } + + @Test + void testCheck_inFieldsMode_ok() { + YamlInputMeta meta = new YamlInputMeta(); + meta.setInFields(true); + meta.setYamlField("yaml"); + + meta.getInputFields().add(new YamlInputField("name")); + + List remarks = new ArrayList<>(); + + meta.check( + remarks, + null, + new TransformMeta(), + null, + new String[] {"input"}, + null, + null, + new Variables(), + null); + + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_OK)); + } + + @Test + void testConvertLegacyXml(@TempDir Path tempDir) throws Exception { + Path tempFile = tempDir.resolve("test.yaml"); + Files.createFile(tempFile); + + String xml = + String.format( + """ + + %s + .*\\.yaml + Y + Y + + """, + tempFile); + Node node = XmlHandler.loadXmlString(xml); + + YamlInputMeta meta = new YamlInputMeta(); + meta.convertLegacyXml(node); + + assertEquals(1, meta.getYamlFiles().size()); + + YamlInputMeta.YamlFile file = meta.getYamlFiles().getFirst(); + assertEquals(tempFile.toString(), file.getFilename()); + assertTrue(file.isFileRequired()); + assertTrue(file.isIncludingSubFolders()); + } } diff --git a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java new file mode 100644 index 00000000000..21877c167c7 --- /dev/null +++ b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputTests.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.yamlinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.FileWriter; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.reflect.MethodUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.fileinput.FileInputList; +import org.apache.hop.core.logging.ILoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.Mockito; + +/** Unit test for {@link YamlInput} */ +class YamlInputTests { + private TransformMockHelper helper; + @TempDir private Path tempDir; + @Mock private IVariables variables; + + @BeforeEach + void setup() throws Exception { + HopEnvironment.init(); + + helper = new TransformMockHelper<>("Yaml_test", YamlInputMeta.class, YamlInputData.class); + Mockito.doReturn(helper.iLogChannel) + .when(helper.logChannelFactory) + .create(any(), any(ILoggingObject.class)); + + when(helper.pipeline.isRunning()).thenReturn(true); + } + + @Test + void testInit_shouldBuildRowMeta() { + YamlInputField field = new YamlInputField(); + field.setPath("user.name"); + field.setType(IValueMeta.TYPE_STRING); + field.setTrimType(YamlInputField.TYPE_TRIM_BOTH); + + YamlInputMeta meta = helper.iTransformMeta; + YamlInputData data = helper.iTransformData; + PipelineMeta plMeta = helper.pipelineMeta; + Pipeline pipeline = helper.pipeline; + YamlInput input = new YamlInput(helper.transformMeta, meta, data, 0, plMeta, pipeline); + + when(meta.getInputFields()).thenReturn(List.of(field)); + + boolean result = input.init(); + assertTrue(result); + + assertNotNull(input.getData().rowMeta); + assertEquals(1, input.getData().nrInputFields); + } + + @Test + void testHandleMissingFiles_shouldThrowException() { + FileObject mockFile = mock(FileObject.class); + FileInputList fileList = mock(FileInputList.class); + + when(fileList.getNonExistentFiles()).thenReturn(List.of(mockFile)); + + YamlInputMeta meta = helper.iTransformMeta; + YamlInputData data = helper.iTransformData; + PipelineMeta plMeta = helper.pipelineMeta; + Pipeline pipeline = helper.pipeline; + data.files = fileList; + YamlInput transform = new YamlInput(helper.transformMeta, meta, data, 0, plMeta, pipeline); + + InvocationTargetException ex = + assertThrows( + InvocationTargetException.class, + () -> MethodUtils.invokeMethod(transform, true, "handleMissingFiles")); + + Throwable exception = ex.getTargetException(); + assertInstanceOf(HopException.class, exception); + } + + @Test + void testGetRowData_shouldMergeYamlAndInputRow() throws Exception { + YamlInputMeta meta = helper.iTransformMeta; + YamlInputData data = helper.iTransformData; + PipelineMeta plMeta = helper.pipelineMeta; + Pipeline pipeline = helper.pipeline; + YamlInput transform = new YamlInput(helper.transformMeta, meta, data, 0, plMeta, pipeline); + + // mock yaml reader + YamlReader yamlReader = mock(YamlReader.class); + Object[] yamlRow = new Object[] {"alice", 18}; + + data.yaml = yamlReader; + data.rowMeta = new RowMeta(); + data.totalPreviousFields = 1; + data.totalOutStreamFields = 3; + data.totalOutFields = 1; + + data.readRow = new Object[] {"file1.yml"}; + when(yamlReader.getRow(any())).thenReturn(yamlRow); + + Object[] result = (Object[]) MethodUtils.invokeMethod(transform, true, "getRowData"); + + assertNotNull(result); + assertEquals("file1.yml", result[0]); + assertEquals("alice", result[1]); + assertEquals(18, result[2]); + } + + @Test + void testOpenNextFile_shouldAdvanceIndex() throws Exception { + YamlInputMeta meta = helper.iTransformMeta; + YamlInputData data = helper.iTransformData; + PipelineMeta plMeta = helper.pipelineMeta; + Pipeline pipeline = helper.pipeline; + YamlInput transform = new YamlInput(helper.transformMeta, meta, data, 0, plMeta, pipeline); + + Path tempFile = tempDir.resolve("test.yaml"); + try (FileWriter writer = new FileWriter(tempFile.toFile())) { + writer.write(""" + name: Alice + age: 30 + """); + } + + data.fileIndex = 0; + + data.files = mock(FileInputList.class); + when(data.files.nrOfFiles()).thenReturn(1); + when(data.files.getFile(0)).thenReturn(HopVfs.getFileObject(tempFile.toString())); + + boolean result = (boolean) MethodUtils.invokeMethod(transform, true, "openNextFile"); + assertTrue(result); + assertEquals(1, data.fileIndex); + assertNotNull(data.yaml); + } + + /** + * {@link YamlInput#processRow()} in file mode: reads a YAML map, emits one row via {@code + * putRow}, then returns false when no more rows. + */ + @Test + void testProcessRow_fileMode_emitsOneRowThenFinishes() throws Exception { + Path tempFile = tempDir.resolve("processRow-one.yaml"); + Files.writeString(tempFile, """ + name: Alice + age: 30 + """); + + YamlInputMeta meta = new YamlInputMeta(); + meta.setInFields(false); + meta.setDoNotFailIfNoFile(true); + YamlInputField fName = new YamlInputField(); + fName.setName("name"); + fName.setPath("name"); + fName.setType(IValueMeta.TYPE_STRING); + YamlInputField fAge = new YamlInputField(); + fAge.setName("age"); + fAge.setPath("age"); + fAge.setType(IValueMeta.TYPE_INTEGER); + meta.getInputFields().add(fName); + meta.getInputFields().add(fAge); + YamlInputMeta.YamlFile yf = new YamlInputMeta.YamlFile(); + yf.setFilename(tempFile.toAbsolutePath().toString()); + yf.setFileMask(".*\\.ya?ml"); + yf.setFileRequired(false); + meta.getYamlFiles().add(yf); + + YamlInputData data = new YamlInputData(); + YamlInput transform = + new YamlInput(helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline); + transform = spy(transform); + + List written = new ArrayList<>(); + doAnswer( + invocation -> { + Object[] row = invocation.getArgument(1); + written.add(Arrays.copyOf(row, row.length)); + return null; + }) + .when(transform) + .putRow(any(IRowMeta.class), any()); + + assertTrue(transform.init()); + assertTrue(transform.processRow(), "first processRow should emit a row"); + assertFalse(transform.processRow(), "second processRow should end (no more rows)"); + assertEquals(1, written.size()); + assertEquals("Alice", written.getFirst()[0]); + assertEquals(30L, ((Number) written.getFirst()[1]).longValue()); + } + + /** + * Row limit stops {@link YamlInput#processRow()} after the configured number of output rows + * (single-document map still yields only one row when limit is 1). + */ + @Test + void testProcessRow_rowLimit_stopsAfterOneRow() throws Exception { + Path tempFile = tempDir.resolve("processRow-limit.yaml"); + Files.writeString(tempFile, """ + name: Bob + age: 40 + """); + + YamlInputMeta meta = new YamlInputMeta(); + meta.setInFields(false); + meta.setDoNotFailIfNoFile(true); + meta.setRowLimit(1); + YamlInputField fName = new YamlInputField(); + fName.setName("name"); + fName.setPath("name"); + fName.setType(IValueMeta.TYPE_STRING); + YamlInputField fAge = new YamlInputField(); + fAge.setName("age"); + fAge.setPath("age"); + fAge.setType(IValueMeta.TYPE_INTEGER); + meta.getInputFields().add(fName); + meta.getInputFields().add(fAge); + YamlInputMeta.YamlFile yf = new YamlInputMeta.YamlFile(); + yf.setFilename(tempFile.toAbsolutePath().toString()); + yf.setFileMask(".*\\.ya?ml"); + yf.setFileRequired(false); + meta.getYamlFiles().add(yf); + + YamlInputData data = new YamlInputData(); + YamlInput transform = + spy( + new YamlInput( + helper.transformMeta, meta, data, 0, helper.pipelineMeta, helper.pipeline)); + + List written = new ArrayList<>(); + doAnswer( + invocation -> { + Object[] row = invocation.getArgument(1); + written.add(Arrays.copyOf(row, row.length)); + return null; + }) + .when(transform) + .putRow(any(IRowMeta.class), any()); + + assertTrue(transform.init()); + assertFalse( + transform.processRow(), + "one row is emitted then processRow returns false when row limit is exceeded"); + assertFalse(transform.processRow(), "no further rows after end of YAML"); + assertEquals(1, written.size()); + assertEquals("Bob", written.getFirst()[0]); + assertEquals(40L, ((Number) written.getFirst()[1]).longValue()); + } + + @AfterEach + void tearDown() { + helper.cleanUp(); + } +} diff --git a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java new file mode 100644 index 00000000000..b105f9f7be5 --- /dev/null +++ b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.yamlinput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.FileWriter; +import java.nio.file.Path; +import java.util.Date; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.vfs.HopVfs; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Unit test for {@link YamlReader} */ +class YamlReaderTests { + private YamlReader reader; + + @BeforeEach + void setUp() { + reader = new YamlReader(); + } + + @BeforeAll + static void init() { + try { + HopEnvironment.init(); + } catch (HopException e) { + throw new RuntimeException(e); + } + } + + @Test + void testLoadString_MapStructure() { + String yaml = """ + name: John + age: 30 + active: true + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + assertEquals(3, rowMeta.size()); + + Object[] row = reader.getRow(rowMeta); + + assertNotNull(row); + assertEquals("John", row[0]); + // Integer -> Long + assertEquals(30L, row[1]); + assertEquals(true, row[2]); + } + + @Test + void testLoadString_ListOfMap() { + String yaml = """ + - name: Alice + age: 25 + createTime: 2026-03-04 12:03:01 + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + Object[] row = reader.getRow(rowMeta); + + assertNotNull(row); + assertEquals("Alice", row[0]); + assertEquals(25L, row[1]); + assertInstanceOf(Date.class, row[2]); + } + + @Test + void testLoadString_ListMultipleValues() { + String yaml = """ + - 1 + - 2 + - 3 + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + // DEFAULT_LIST_VALUE_NAME + assertEquals(1, rowMeta.size()); + + Object[] row1 = reader.getRow(rowMeta); + Object[] row2 = reader.getRow(rowMeta); + Object[] row3 = reader.getRow(rowMeta); + + assertEquals(1L, row1[0]); + assertEquals(2L, row2[0]); + assertEquals(3L, row3[0]); + } + + @Test + void testMultipleDocuments() { + String yaml = """ + --- + name: A + --- + name: B + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + Object[] row1 = reader.getRow(rowMeta); + Object[] row2 = reader.getRow(rowMeta); + + assertEquals("A", row1[0]); + assertEquals("B", row2[0]); + } + + @Test + void testTypeConversion_Number() { + String yaml = """ + value: 12.5 + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + Object[] row = reader.getRow(rowMeta); + + assertInstanceOf(Double.class, row[0]); + assertEquals(12.5, (Double) row[0]); + } + + @Test + void testNestedMapToString() { + String yaml = """ + person: + name: Tom + age: 20 + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + Object[] row = reader.getRow(rowMeta); + + assertNotNull(row[0]); + assertTrue(row[0].toString().contains("name")); + assertTrue(row[0].toString().contains("age")); + } + + @Test + void testEmptyYaml() { + reader.loadString(""); + + RowMeta rowMeta = reader.getFields(); + Object[] row = reader.getRow(rowMeta); + + assertEquals(0, row.length); + } + + @Test + void testGetFields_TypeInference() { + String yaml = + """ + - intField: 1 + longField: 2 + doubleField: 1.5 + boolField: true + bigIntField: 99999999999999999999 + bigDecField: 9999999999999999999999999.55 + byteField: 1 + dateField: 2026-03-04T12:03:01 + """; + + reader.loadString(yaml); + + RowMeta rowMeta = reader.getFields(); + assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.getValueMeta(0).getType()); + assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.getValueMeta(1).getType()); + assertEquals(IValueMeta.TYPE_NUMBER, rowMeta.getValueMeta(2).getType()); + assertEquals(IValueMeta.TYPE_BOOLEAN, rowMeta.getValueMeta(3).getType()); + assertEquals(IValueMeta.TYPE_BIGNUMBER, rowMeta.getValueMeta(4).getType()); + assertEquals(IValueMeta.TYPE_NUMBER, rowMeta.getValueMeta(5).getType()); + assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.getValueMeta(6).getType()); + assertEquals(IValueMeta.TYPE_DATE, rowMeta.getValueMeta(7).getType()); + } + + @Test + void testLoadFile_singleDocument_map(@TempDir Path tempDir) throws Exception { + Path tempFile = tempDir.resolve("test.yaml"); + try (FileWriter writer = new FileWriter(tempFile.toFile())) { + writer.write(""" + name: Alice + age: 30 + """); + } + + FileObject fileObject = HopVfs.getFileObject(tempFile.toString()); + reader.loadFile(fileObject); + + assertNotNull(reader.getYaml()); + assertTrue(reader.isMapUsed()); + + RowMeta rowMeta = reader.getFields(); + assertEquals(2, rowMeta.size()); + assertTrue(reader.getBytesReadFromFile() > 0); + } +}