From 3d3f51d66ac37e4807d7311fef6033426c3bcd76 Mon Sep 17 00:00:00 2001 From: Matt Casters Date: Thu, 1 Dec 2022 14:48:52 +0100 Subject: [PATCH] Fix #1860 : Transform Get Subfolders is broken HOP-4308 : Cleanup XML of transform Get Sub Folders --- .../hop/core/fileinput/FileInputList.java | 151 +++--- .../0012-get-subfolder-names-validation.hpl | 179 +++++++ .../0012-get-subfolder-names.hpl | 477 +++++++++++++++++ .../datasets/0012-get-subfolder-names.csv | 25 + .../main-0012-get-subfolder-names.hwf | 139 +++++ .../dataset/0012-get-subfolder-names.json | 24 + ...2-get-subfolder-names-validation UNIT.json | 32 ++ .../transforms/0060-get-subfolder-names.hpl | 446 ++++++++++++++++ .../datasets/golden-get-subfolder-names.csv | 27 + .../main-0060-get-subfolder-names.hwf | 79 +++ .../dataset/golden-get-subfolder-names.json | 24 + .../0060-get-subfolder-names UNIT.json | 32 ++ .../getsubfolders/GetSubFolders.java | 225 ++++---- .../getsubfolders/GetSubFoldersData.java | 22 +- .../getsubfolders/GetSubFoldersDialog.java | 480 +++++++++-------- .../getsubfolders/GetSubFoldersMeta.java | 500 +++++++++--------- .../getsubfolders/GetSubFoldersMetaTest.java | 147 +---- .../get-subfolder-names-transform.xml | 29 + 18 files changed, 2196 insertions(+), 842 deletions(-) create mode 100644 integration-tests/beam_directrunner/0012-get-subfolder-names-validation.hpl create mode 100644 integration-tests/beam_directrunner/0012-get-subfolder-names.hpl create mode 100644 integration-tests/beam_directrunner/datasets/0012-get-subfolder-names.csv create mode 100644 integration-tests/beam_directrunner/main-0012-get-subfolder-names.hwf create mode 100644 integration-tests/beam_directrunner/metadata/dataset/0012-get-subfolder-names.json create mode 100644 integration-tests/beam_directrunner/metadata/unit-test/0012-get-subfolder-names-validation UNIT.json create mode 100644 integration-tests/transforms/0060-get-subfolder-names.hpl create mode 100644 integration-tests/transforms/datasets/golden-get-subfolder-names.csv create mode 100644 integration-tests/transforms/main-0060-get-subfolder-names.hwf create mode 100644 integration-tests/transforms/metadata/dataset/golden-get-subfolder-names.json create mode 100644 integration-tests/transforms/metadata/unit-test/0060-get-subfolder-names UNIT.json create mode 100644 plugins/transforms/getsubfolders/src/test/resources/get-subfolder-names-transform.xml diff --git a/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java b/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java index 8d11659684b..2c63403e840 100644 --- a/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java +++ b/core/src/main/java/org/apache/hop/core/fileinput/FileInputList.java @@ -34,7 +34,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; @@ -50,10 +49,10 @@ public enum FileTypeFilter { ONLY_FILES("only_files", FileType.FILE), ONLY_FOLDERS("only_folders", FileType.FOLDER); - private String name; + private final String name; private final Collection allowedFileTypes; - private FileTypeFilter(String name, FileType... allowedFileTypes) { + FileTypeFilter(String name, FileType... allowedFileTypes) { this.name = name; this.allowedFileTypes = Collections.unmodifiableCollection(Arrays.asList(allowedFileTypes)); } @@ -90,20 +89,19 @@ public static FileTypeFilter getByName(String name) { public static String getRequiredFilesDescription(List nonExistantFiles) { StringBuilder buffer = new StringBuilder(); - for (Iterator iter = nonExistantFiles.iterator(); iter.hasNext(); ) { - FileObject file = iter.next(); + for (FileObject file : nonExistantFiles) { buffer.append(file.getName().getURI()); buffer.append(Const.CR); } return buffer.toString(); } - private static boolean[] includeSubdirsFalse(int iLength) { - boolean[] includeSubdirs = new boolean[iLength]; + private static boolean[] includeSubDirsFalse(int iLength) { + boolean[] includeSubDirs = new boolean[iLength]; for (int i = 0; i < iLength; i++) { - includeSubdirs[i] = false; + includeSubDirs[i] = false; } - return includeSubdirs; + return includeSubDirs; } public static String[] createFilePathList( @@ -112,9 +110,9 @@ public static String[] createFilePathList( String[] fileMask, String[] excludeFileMask, String[] fileRequired) { - boolean[] includeSubdirs = includeSubdirsFalse(fileName.length); + boolean[] includeSubDirs = includeSubDirsFalse(fileName.length); return createFilePathList( - variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null); + variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubDirs, null); } public static String[] createFilePathList( @@ -123,9 +121,9 @@ public static String[] createFilePathList( String[] fileMask, String[] excludeFileMask, String[] fileRequired, - boolean[] includeSubdirs) { + boolean[] includeSubDirs) { return createFilePathList( - variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null); + variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubDirs, null); } public static String[] createFilePathList( @@ -134,7 +132,7 @@ public static String[] createFilePathList( String[] fileMask, String[] excludeFileMask, String[] fileRequired, - boolean[] includeSubdirs, + boolean[] includeSubDirs, FileTypeFilter[] filters) { List fileList = createFileList( @@ -143,7 +141,7 @@ public static String[] createFilePathList( fileMask, excludeFileMask, fileRequired, - includeSubdirs, + includeSubDirs, filters) .getFiles(); String[] filePaths = new String[fileList.size()]; @@ -159,9 +157,9 @@ public static FileInputList createFileList( String[] fileMask, String[] excludeFileMask, String[] fileRequired) { - boolean[] includeSubdirs = includeSubdirsFalse(fileName.length); + boolean[] includeSubDirs = includeSubDirsFalse(fileName.length); return createFileList( - variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null); + variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubDirs, null); } public static FileInputList createFileList( @@ -170,9 +168,9 @@ public static FileInputList createFileList( String[] fileMask, String[] excludeFileMask, String[] fileRequired, - boolean[] includeSubdirs) { + boolean[] includeSubDirs) { return createFileList( - variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null); + variables, fileName, fileMask, excludeFileMask, fileRequired, includeSubDirs, null); } public static FileInputList createFileList( @@ -181,7 +179,7 @@ public static FileInputList createFileList( String[] fileMask, String[] excludeFileMask, String[] fileRequired, - boolean[] includeSubdirs, + boolean[] includeSubDirs, FileTypeFilter[] fileTypeFilters) { FileInputList fileInputList = new FileInputList(); @@ -195,7 +193,7 @@ public static FileInputList createFileList( final String oneMask = realMask[i]; final String excludeOneMask = realExcludeMask[i]; final boolean oneRequired = YES.equalsIgnoreCase(fileRequired[i]); - final boolean subDirs = includeSubdirs[i]; + final boolean subDirs = includeSubDirs[i]; final FileTypeFilter filter = ((fileTypeFilters == null || fileTypeFilters[i] == null) ? FileTypeFilter.ONLY_FILES @@ -264,8 +262,7 @@ public boolean includeFile(FileSelectInfo info) { } }); if (fileObjects != null) { - for (int j = 0; j < fileObjects.length; j++) { - FileObject fileObject = fileObjects[j]; + for (FileObject fileObject : fileObjects) { if (fileObject.exists()) { fileInputList.addFile(fileObject); } @@ -280,19 +277,19 @@ public boolean includeFile(FileSelectInfo info) { fileInputList.sortFiles(); } else if (directoryFileObject instanceof CompressedFileFileObject) { FileObject[] children = directoryFileObject.getChildren(); - for (int j = 0; j < children.length; j++) { + for (FileObject child : children) { // See if the wildcard (regexp) matches... - String name = children[j].getName().getBaseName(); + String name = child.getName().getBaseName(); boolean matches = true; if (!Utils.isEmpty(oneMask)) { matches = Pattern.matches(oneMask, name); } - boolean excludematches = false; + boolean excludeMatches = false; if (!Utils.isEmpty(excludeOneMask)) { - excludematches = Pattern.matches(excludeOneMask, name); + excludeMatches = Pattern.matches(excludeOneMask, name); } - if (matches && !excludematches) { - fileInputList.addFile(children[j]); + if (matches && !excludeMatches) { + fileInputList.addFile(child); } } // We don't sort here, keep the order of the files in the archive. @@ -329,83 +326,63 @@ public static FileInputList createFolderList( FileInputList fileInputList = new FileInputList(); // Replace possible environment variables... - final String[] realfolder = variables.resolve(folderName); + final String[] realFolder = variables.resolve(folderName); - for (int i = 0; i < realfolder.length; i++) { - final String onefile = realfolder[i]; - final boolean onerequired = YES.equalsIgnoreCase(folderRequired[i]); - final boolean subdirs = true; + for (int i = 0; i < realFolder.length; i++) { + final String oneFile = realFolder[i]; + final boolean oneRequired = YES.equalsIgnoreCase(folderRequired[i]); final FileTypeFilter filter = FileTypeFilter.ONLY_FOLDERS; - if (Utils.isEmpty(onefile)) { + if (Utils.isEmpty(oneFile)) { continue; } - FileObject directoryFileObject = null; - try { + try (FileObject directoryFileObject = HopVfs.getFileObject(oneFile)) { // Find all folder names in this directory // - directoryFileObject = HopVfs.getFileObject(onefile); if (directoryFileObject != null - && directoryFileObject.getType() == FileType.FOLDER) { // it's a directory + && directoryFileObject.getType() == FileType.FOLDER) { // it's a directory FileObject[] fileObjects = - directoryFileObject.findFiles( - new AllFileSelector() { - @Override - public boolean traverseDescendents(FileSelectInfo info) { - return info.getDepth() == 0 || subdirs; - } - - @Override - public boolean includeFile(FileSelectInfo info) { - // Never return the parent directory of a file list. - if (info.getDepth() == 0) { - return false; - } - - FileObject fileObject = info.getFile(); - try { - if (fileObject != null && filter.isFileTypeAllowed(fileObject.getType())) { - return true; - } - return false; - } catch (IOException ex) { - // Upon error don't process the file. - return false; - } - } - }); + directoryFileObject.findFiles( + new AllFileSelector() { + @Override + public boolean includeFile(FileSelectInfo info) { + // Never return the parent directory of a file list. + if (info.getDepth() == 0) { + return false; + } + + FileObject fileObject = info.getFile(); + try { + return fileObject != null && filter.isFileTypeAllowed(fileObject.getType()); + } catch (IOException ex) { + // Upon error don't process the file. + return false; + } + } + }); if (fileObjects != null) { - for (int j = 0; j < fileObjects.length; j++) { - if (fileObjects[j].exists()) { - fileInputList.addFile(fileObjects[j]); + for (FileObject fileObject : fileObjects) { + if (fileObject.exists()) { + fileInputList.addFile(fileObject); } } } - if (Utils.isEmpty(fileObjects) && onerequired) { - + if (Utils.isEmpty(fileObjects) && oneRequired) { fileInputList.addNonAccessibleFile(directoryFileObject); } // Sort the list: quicksort, only for regular files fileInputList.sortFiles(); } else { - if (onerequired && !directoryFileObject.exists()) { + if (oneRequired && (directoryFileObject == null || !directoryFileObject.exists())) { fileInputList.addNonExistantFile(directoryFileObject); } } } catch (Exception e) { log.logError(Const.getStackTracker(e)); - } finally { - try { - if (directoryFileObject != null) { - directoryFileObject.close(); - } - directoryFileObject = null; - } catch (Exception e) { - // Ignore - } } + // Ignore } return fileInputList; @@ -452,9 +429,9 @@ public void addNonExistantFile(FileObject file) { } public void sortFiles() { - Collections.sort(files, HopVfs.getComparator()); - Collections.sort(nonAccessibleFiles, HopVfs.getComparator()); - Collections.sort(nonExistentFiles, HopVfs.getComparator()); + files.sort(HopVfs.getComparator()); + nonAccessibleFiles.sort(HopVfs.getComparator()); + nonExistentFiles.sort(HopVfs.getComparator()); } public FileObject getFile(int i) { @@ -474,27 +451,27 @@ public static FileInputList createFileList( String[] fileName, String[] fileMask, String[] fileRequired, - boolean[] includeSubdirs) { + boolean[] includeSubDirs) { return createFileList( variables, fileName, fileMask, new String[fileName.length], fileRequired, - includeSubdirs, + includeSubDirs, null); } public static String[] createFilePathList( IVariables variables, String[] fileName, String[] fileMask, String[] fileRequired) { - boolean[] includeSubdirs = includeSubdirsFalse(fileName.length); + boolean[] includeSubDirs = includeSubDirsFalse(fileName.length); return createFilePathList( variables, fileName, fileMask, new String[fileName.length], fileRequired, - includeSubdirs, + includeSubDirs, null); } } diff --git a/integration-tests/beam_directrunner/0012-get-subfolder-names-validation.hpl b/integration-tests/beam_directrunner/0012-get-subfolder-names-validation.hpl new file mode 100644 index 00000000000..b7bfc33c225 --- /dev/null +++ b/integration-tests/beam_directrunner/0012-get-subfolder-names-validation.hpl @@ -0,0 +1,179 @@ + + + + + 0012-get-subfolder-names-validation + Y + + + + Normal + + + N + 1000 + 100 + - + 2022/12/01 12:53:14.315 + - + 2022/12/01 12:53:14.315 + + + + + + /tmp/0012-get-subfolder-names*.csv + Verify + Y + + + + /tmp/0012-get-subfolder-names*.csv + TextFileInput2 + + Y + + 1 + + none + + + N + N + + + , + " + N + +
N
+ 1 +
N
+ 1 + N + 1 + N + 80 + 0 + Y + N + + N + N + + Unix + UTF-8 + Characters + Y + + ${java.io.tmpdir}/0012/ + get-subfolder-names.*\.csv$ + + N + N + CSV + None + + + + + + id + Integer + # + $ + . + , + - + + -1 + 15 + 0 + none + N + + + shortFolder + String + + $ + . + , + - + + -1 + 26 + -1 + none + N + + + 0 + N + N + + + N + + + + + warning + + error + + line + Y + en_US + + + + + + + + + + + 144 + 80 + + + + Verify + Dummy + + Y + + 1 + + none + + + + + 432 + 80 + + + + + + diff --git a/integration-tests/beam_directrunner/0012-get-subfolder-names.hpl b/integration-tests/beam_directrunner/0012-get-subfolder-names.hpl new file mode 100644 index 00000000000..13178035aa3 --- /dev/null +++ b/integration-tests/beam_directrunner/0012-get-subfolder-names.hpl @@ -0,0 +1,477 @@ + + + + + 0012-get-subfolder-names + Y + + + + Normal + + + N + 1000 + 100 + - + 2022/12/01 12:49:29.477 + - + 2022/12/01 12:49:29.477 + + + + + + Get subfolder names + folderName, rowNumber only + Y + + + Select values + lookup folder + Y + + + folderName, rowNumber only + projectHome + Y + + + folders + lookup folder + Y + + + lookup folder + only found + Y + + + projectHome + shortName + Y + + + shortName + Select values + Y + + + only found + limit + Y + + + limit + /tmp/0012/get-subfolder-names*.csv + Y + + + + /tmp/0012/get-subfolder-names*.csv + BeamOutput + + Y + + 1 + + none + + + get-subfolder-names + .csv + ${java.io.tmpdir}/0012/ + N + + + 944 + 176 + + + + Get subfolder names + GetSubFolders + + Y + + 1 + + none + + + Y + N + rowNumber + + 0 + + ${PROJECT_HOME} + Y + + + + 112 + 64 + + + + Select values + SelectValues + + Y + + 1 + + none + + + + + rowNumber + + + shortName + + N + + + + 336 + 176 + + + + folderName, rowNumber only + SelectValues + + Y + + 1 + + none + + + + + folderName + + + rowNumber + + N + + + + 336 + 64 + + + + folders + DataGrid + + Y + + 1 + + none + + + + + metadata + + + metadata/workflow-run-configuration + + + metadata/cassandra-connection + + + metadata/execution-info-location + + + metadata/rdbms + + + metadata/pipeline-probe + + + metadata/neo4j-connection + + + metadata/unit-test + + + metadata/splunk + + + metadata/partition + + + metadata/neo4j-graph-model + + + metadata/dataset + + + metadata/server + + + metadata/file-definition + + + metadata/pipeline-run-configuration + + + metadata/async-web-service + + + metadata/execution-data-profile + + + metadata/mongodb-connection + + + metadata/web-service + + + metadata/workflow-log + + + metadata/pipeline-log + + + input + + + datasets + + + output + + + + + -1 + -1 + N + folder + String + + + + + 400 + 288 + + + + limit + SelectValues + + Y + + 1 + + none + + + + + rowNumber + + + shortName + + N + + + + 736 + 176 + + + + lookup folder + StreamLookup + + Y + + 1 + + none + + + folders + N + N + N + N + + + shortName + folder + + + folder + found + + String + + + + + 496 + 176 + + + + only found + FilterRows + + Y + + 1 + + none + + + + + + + IS NOT NULL + found + N + - + + + /tmp/not-found*.csv + limit + + + 624 + 176 + + + + projectHome + GetVariable + + Y + + 1 + + none + + + + + projectHome + ${PROJECT_HOME} + String + + + + + -1 + -1 + none + + + + + 544 + 64 + + + + shortName + ScriptValueMod + + Y + + 1 + + none + + + 9 + + + 0 + Script 1 + +var shortName = replace(folderName, projectHome+"/", ""); + + + + + -1 + replace_Sample + // Replaces Strings inside the given Variable. + // It is possible to make more replacements inside + // one call. The given variables must be an odd number + // + // Usage: + // replace(var, var, var); + // 1: String - The Variable with the content to replace. + // 2: String - The Value to search for. + // 3: String - The Value to replace with. + // + // replace(var, var, var, var,var,...) + // 1: String - The Variable with the content to replace. + // 2: String - The First Value to search for. + // 3: String - The First Value to replace with. + // 4: String - The Second Value to search for. + // 5: String - The Second Value to replace with. + // ... + // + // 2006-11-15 + // + var str1 = "Hello World, this is a nice function"; + var str2 = replace(str1,"World", "Folk"); + Alert(str2); + var str2 = replace(str1,"World", "Folk", "nice","beautifull"); + Alert(str2); + + + + + + shortName + shortName + String + -1 + -1 + N + + + + + 672 + 64 + + + + + + diff --git a/integration-tests/beam_directrunner/datasets/0012-get-subfolder-names.csv b/integration-tests/beam_directrunner/datasets/0012-get-subfolder-names.csv new file mode 100644 index 00000000000..03fe83e37b7 --- /dev/null +++ b/integration-tests/beam_directrunner/datasets/0012-get-subfolder-names.csv @@ -0,0 +1,25 @@ +id,shortFolder +1,datasets +2,input +3,metadata +4,metadata/async-web-service +5,metadata/cassandra-connection +6,metadata/dataset +7,metadata/execution-data-profile +8,metadata/execution-info-location +9,metadata/file-definition +10,metadata/mongodb-connection +11,metadata/neo4j-connection +12,metadata/neo4j-graph-model +13,metadata/partition +14,metadata/pipeline-log +15,metadata/pipeline-probe +16,metadata/pipeline-run-configuration +17,metadata/rdbms +18,metadata/server +19,metadata/splunk +20,metadata/unit-test +21,metadata/web-service +22,metadata/workflow-log +23,metadata/workflow-run-configuration +24,output diff --git a/integration-tests/beam_directrunner/main-0012-get-subfolder-names.hwf b/integration-tests/beam_directrunner/main-0012-get-subfolder-names.hwf new file mode 100644 index 00000000000..a0ddeff0d7e --- /dev/null +++ b/integration-tests/beam_directrunner/main-0012-get-subfolder-names.hwf @@ -0,0 +1,139 @@ + + + + main-0012-get-subfolder-names + Y + + + + - + 2022/12/01 12:52:32.774 + - + 2022/12/01 12:52:32.774 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 80 + 80 + + + + 0012-get-subfolder-names + + PIPELINE + + ${PROJECT_HOME}/0012-get-subfolder-names.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + local + + Y + + N + 512 + 80 + + + + Delete /tmp/0012/*.csv + + DELETE_FILES + + N + N + + + ${java.io.tmpdir}/0012/ + get-subfolder-names-.*\.csv$ + + + N + 272 + 80 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0012-get-subfolder-names-validation UNIT + + + N + 752 + 80 + + + + + + Start + Delete /tmp/0012/*.csv + Y + Y + Y + + + Delete /tmp/0012/*.csv + 0012-get-subfolder-names + Y + Y + N + + + 0012-get-subfolder-names + Run Pipeline Unit Tests + Y + Y + N + + + + + + diff --git a/integration-tests/beam_directrunner/metadata/dataset/0012-get-subfolder-names.json b/integration-tests/beam_directrunner/metadata/dataset/0012-get-subfolder-names.json new file mode 100644 index 00000000000..f73f86a9ea1 --- /dev/null +++ b/integration-tests/beam_directrunner/metadata/dataset/0012-get-subfolder-names.json @@ -0,0 +1,24 @@ +{ + "base_filename": "0012-get-subfolder-names.csv", + "name": "0012-get-subfolder-names", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": -1, + "field_name": "id", + "field_format": "#" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "shortFolder", + "field_format": "" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/beam_directrunner/metadata/unit-test/0012-get-subfolder-names-validation UNIT.json b/integration-tests/beam_directrunner/metadata/unit-test/0012-get-subfolder-names-validation UNIT.json new file mode 100644 index 00000000000..ea29c6bda20 --- /dev/null +++ b/integration-tests/beam_directrunner/metadata/unit-test/0012-get-subfolder-names-validation UNIT.json @@ -0,0 +1,32 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "shortFolder", + "data_set_field": "shortFolder" + } + ], + "field_order": [ + "id" + ], + "data_set_name": "0012-get-subfolder-names", + "transform_name": "Verify" + } + ], + "input_data_sets": [], + "name": "0012-get-subfolder-names-validation UNIT", + "description": "", + "persist_filename": "", + "trans_test_tweaks": [], + "pipeline_filename": "./0012-get-subfolder-names-validation.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file diff --git a/integration-tests/transforms/0060-get-subfolder-names.hpl b/integration-tests/transforms/0060-get-subfolder-names.hpl new file mode 100644 index 00000000000..5a954d56da0 --- /dev/null +++ b/integration-tests/transforms/0060-get-subfolder-names.hpl @@ -0,0 +1,446 @@ + + + + + 0060-get-subfolder-names + Y + + + + Normal + + + N + 1000 + 100 + - + 2022/12/01 12:22:20.526 + - + 2022/12/01 12:22:20.526 + + + + + + projectHome + shortName + Y + + + Get subfolder names + folderName, rowNumber only + Y + + + folderName, rowNumber only + projectHome + Y + + + shortName + Select values + Y + + + Select values + lookup folder + Y + + + folders + lookup folder + Y + + + lookup folder + only found + Y + + + only found + validate + Y + + + + Get subfolder names + GetSubFolders + + Y + + 1 + + none + + + Y + N + rowNumber + + 0 + + ${PROJECT_HOME} + Y + + + + 112 + 48 + + + + Select values + SelectValues + + Y + + 1 + + none + + + + + rowNumber + + + shortName + + N + + + + 336 + 160 + + + + folderName, rowNumber only + SelectValues + + Y + + 1 + + none + + + + + folderName + + + rowNumber + + N + + + + 336 + 48 + + + + folders + DataGrid + + Y + + 1 + + none + + + + + datasets + + + files + + + files/excel + + + metadata + + + metadata/async-web-service + + + metadata/cassandra-connection + + + metadata/dataset + + + metadata/execution-data-profile + + + metadata/execution-info-location + + + metadata/file-definition + + + metadata/git + + + metadata/log-reader + + + metadata/mongodb-connection + + + metadata/neo4j-connection + + + metadata/neo4j-graph-model + + + metadata/partition + + + metadata/pipeline-log + + + metadata/pipeline-probe + + + metadata/pipeline-run-configuration + + + metadata/rdbms + + + metadata/server + + + metadata/splunk + + + metadata/unit-test + + + metadata/web-service + + + metadata/workflow-log + + + metadata/workflow-run-configuration + + + + + -1 + -1 + N + folder + String + + + + + 432 + 272 + + + + lookup folder + StreamLookup + + Y + + 1 + + none + + + folders + N + N + N + N + + + shortName + folder + + + folder + found + + String + + + + + 496 + 160 + + + + only found + FilterRows + + Y + + 1 + + none + + + + + + + IS NOT NULL + found + N + - + + + + + 624 + 160 + + + + projectHome + GetVariable + + Y + + 1 + + none + + + + + projectHome + ${PROJECT_HOME} + String + + + + + -1 + -1 + none + + + + + 544 + 48 + + + + shortName + ScriptValueMod + + Y + + 1 + + none + + + 9 + + + 0 + Script 1 + +var shortName = replace(folderName, projectHome+"/", ""); + + + + + -1 + replace_Sample + // Replaces Strings inside the given Variable. + // It is possible to make more replacements inside + // one call. The given variables must be an odd number + // + // Usage: + // replace(var, var, var); + // 1: String - The Variable with the content to replace. + // 2: String - The Value to search for. + // 3: String - The Value to replace with. + // + // replace(var, var, var, var,var,...) + // 1: String - The Variable with the content to replace. + // 2: String - The First Value to search for. + // 3: String - The First Value to replace with. + // 4: String - The Second Value to search for. + // 5: String - The Second Value to replace with. + // ... + // + // 2006-11-15 + // + var str1 = "Hello World, this is a nice function"; + var str2 = replace(str1,"World", "Folk"); + Alert(str2); + var str2 = replace(str1,"World", "Folk", "nice","beautifull"); + Alert(str2); + + + + + + shortName + shortName + String + -1 + -1 + N + + + + + 672 + 48 + + + + validate + Dummy + + Y + + 1 + + none + + + + + 752 + 160 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-get-subfolder-names.csv b/integration-tests/transforms/datasets/golden-get-subfolder-names.csv new file mode 100644 index 00000000000..866dafe0c88 --- /dev/null +++ b/integration-tests/transforms/datasets/golden-get-subfolder-names.csv @@ -0,0 +1,27 @@ +rowNumber,shortName +1,datasets +2,files +3,files/excel +4,metadata +5,metadata/async-web-service +6,metadata/cassandra-connection +7,metadata/dataset +8,metadata/execution-data-profile +9,metadata/execution-info-location +10,metadata/file-definition +11,metadata/git +12,metadata/log-reader +13,metadata/mongodb-connection +14,metadata/neo4j-connection +15,metadata/neo4j-graph-model +16,metadata/partition +17,metadata/pipeline-log +18,metadata/pipeline-probe +19,metadata/pipeline-run-configuration +20,metadata/rdbms +21,metadata/server +22,metadata/splunk +23,metadata/unit-test +24,metadata/web-service +25,metadata/workflow-log +26,metadata/workflow-run-configuration diff --git a/integration-tests/transforms/main-0060-get-subfolder-names.hwf b/integration-tests/transforms/main-0060-get-subfolder-names.hwf new file mode 100644 index 00000000000..e04fa1b4216 --- /dev/null +++ b/integration-tests/transforms/main-0060-get-subfolder-names.hwf @@ -0,0 +1,79 @@ + + + + main-0060-get-subfolder-names + Y + + + + - + 2022/12/01 12:48:14.405 + - + 2022/12/01 12:48:14.405 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 112 + 80 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0060-get-subfolder-names UNIT + + + N + 304 + 80 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-get-subfolder-names.json b/integration-tests/transforms/metadata/dataset/golden-get-subfolder-names.json new file mode 100644 index 00000000000..866fe595e9e --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-get-subfolder-names.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-get-subfolder-names.csv", + "name": "golden-get-subfolder-names", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": 10, + "field_type": 5, + "field_precision": 0, + "field_name": "rowNumber", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "shortName", + "field_format": "" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/unit-test/0060-get-subfolder-names UNIT.json b/integration-tests/transforms/metadata/unit-test/0060-get-subfolder-names UNIT.json new file mode 100644 index 00000000000..b64dc9d87f3 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0060-get-subfolder-names UNIT.json @@ -0,0 +1,32 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "rowNumber", + "data_set_field": "rowNumber" + }, + { + "transform_field": "shortName", + "data_set_field": "shortName" + } + ], + "field_order": [ + "rowNumber" + ], + "data_set_name": "golden-get-subfolder-names", + "transform_name": "validate" + } + ], + "input_data_sets": [], + "name": "0060-get-subfolder-names UNIT", + "description": "", + "persist_filename": "", + "trans_test_tweaks": [], + "pipeline_filename": "./0060-get-subfolder-names.hpl", + "test_type": "UNIT_TEST" +} \ No newline at end of file diff --git a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFolders.java b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFolders.java index 9a0f900be15..4352d1f7232 100644 --- a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFolders.java +++ b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFolders.java @@ -18,7 +18,6 @@ package org.apache.hop.pipeline.transforms.getsubfolders; import org.apache.commons.vfs2.FileObject; -import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.fileinput.FileInputList; @@ -32,7 +31,6 @@ import org.apache.hop.pipeline.transform.BaseTransform; import org.apache.hop.pipeline.transform.TransformMeta; -import java.io.IOException; import java.util.Date; import java.util.List; @@ -41,7 +39,6 @@ * more output streams. */ public class GetSubFolders extends BaseTransform { - private static final Class PKG = GetSubFoldersMeta.class; // For Translator public GetSubFolders( @@ -54,181 +51,171 @@ public GetSubFolders( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } - /** - * Build an empty row based on the meta-data... - * - * @return - */ - private Object[] buildEmptyRow() { - Object[] rowData = RowDataUtil.allocateRowData(data.outputRowMeta.size()); - - return rowData; - } - @Override public boolean processRow() throws HopException { - - if (meta.isFoldernameDynamic() && (data.filenr >= data.filessize)) { + if (meta.isFolderNameDynamic() && (data.fileIndex >= data.filesCount)) { // Grab one row from previous transform ... - data.readrow = getRow(); + data.inputRow = getRow(); } if (first) { first = false; + initialize(); + } - if (meta.isFoldernameDynamic()) { - data.inputRowMeta = getInputRowMeta(); - data.outputRowMeta = data.inputRowMeta.clone(); - meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - - // Get total previous fields - data.totalpreviousfields = data.inputRowMeta.size(); - - // Check is filename field is provided - if (Utils.isEmpty(meta.getDynamicFoldernameField())) { - logError(BaseMessages.getString(PKG, "GetSubFolders.Log.NoField")); - throw new HopException(BaseMessages.getString(PKG, "GetSubFolders.Log.NoField")); - } - - // cache the position of the field - if (data.indexOfFoldernameField < 0) { - String realDynamicFoldername = resolve(meta.getDynamicFoldernameField()); - data.indexOfFoldernameField = data.inputRowMeta.indexOfValue(realDynamicFoldername); - if (data.indexOfFoldernameField < 0) { - // The field is unreachable ! - logError( - BaseMessages.getString(PKG, "GetSubFolders.Log.ErrorFindingField") - + "[" - + realDynamicFoldername - + "]"); - throw new HopException( - BaseMessages.getString( - PKG, "GetSubFolders.Exception.CouldnotFindField", realDynamicFoldername)); - } - } - } else { - // Create the output row meta-data - data.outputRowMeta = new RowMeta(); - meta.getFields( - data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); // get the - // metadata - // populated - - data.files = meta.getFolderList(this); - data.filessize = data.files.nrOfFiles(); - handleMissingFiles(); - } - data.nrTransformFields = data.outputRowMeta.size(); - } // end if first - if (meta.isFoldernameDynamic()) { - if (data.readrow == null) { + if (meta.isFolderNameDynamic()) { + if (data.inputRow == null) { setOutputDone(); return false; } } else { - if (data.filenr >= data.filessize) { + if (data.fileIndex >= data.filesCount) { setOutputDone(); return false; } } + if (outputNextFile()) { + return false; + } + + data.fileIndex++; + + if (checkFeedback(getLinesInput()) && log.isBasic()) { + logBasic(BaseMessages.getString(PKG, "GetSubFolders.Log.NrLine", "" + getLinesInput())); + } + + return true; + } + + private boolean outputNextFile() throws HopTransformException { try { - Object[] outputRow = buildEmptyRow(); + Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size()); int outputIndex = 0; Object[] extraData = new Object[data.nrTransformFields]; - if (meta.isFoldernameDynamic()) { - if (data.filenr >= data.filessize) { + if (meta.isFolderNameDynamic()) { + if (data.fileIndex >= data.filesCount) { // Get value of dynamic filename field ... - String filename = getInputRowMeta().getString(data.readrow, data.indexOfFoldernameField); + String filename = getInputRowMeta().getString(data.inputRow, data.indexOfFolderNameField); + + String[] filesNames = {filename}; + String[] filesRequired = {"N"}; - String[] filesname = {filename}; - String[] filesrequired = {GetSubFoldersMeta.NO}; // Get files list - data.files = meta.getDynamicFolderList(this, filesname, filesrequired); - data.filessize = data.files.nrOfFiles(); - data.filenr = 0; + data.files = meta.getDynamicFolderList(this, filesNames, filesRequired); + data.filesCount = data.files.nrOfFiles(); + data.fileIndex = 0; } // Clone current input row - outputRow = data.readrow.clone(); + outputRow = data.inputRow.clone(); } - if (data.filessize > 0) { - data.file = data.files.getFile(data.filenr); + if (data.filesCount > 0) { + data.file = data.files.getFile(data.fileIndex); // filename extraData[outputIndex++] = HopVfs.getFilename(data.file); - // short_filename + // short filename extraData[outputIndex++] = data.file.getName().getBaseName(); - try { - // Path - extraData[outputIndex++] = HopVfs.getFilename(data.file.getParent()); + // Path + extraData[outputIndex++] = HopVfs.getFilename(data.file.getParent()); - // ishidden - extraData[outputIndex++] = Boolean.valueOf(data.file.isHidden()); + // is hidden? + extraData[outputIndex++] = data.file.isHidden(); - // isreadable - extraData[outputIndex++] = Boolean.valueOf(data.file.isReadable()); + // is readable? + extraData[outputIndex++] = data.file.isReadable(); - // iswriteable - extraData[outputIndex++] = Boolean.valueOf(data.file.isWriteable()); + // is writeable? + extraData[outputIndex++] = data.file.isWriteable(); - // lastmodifiedtime - extraData[outputIndex++] = new Date(data.file.getContent().getLastModifiedTime()); - - } catch (IOException e) { - throw new HopException(e); - } + // last modified time + extraData[outputIndex++] = new Date(data.file.getContent().getLastModifiedTime()); // uri extraData[outputIndex++] = data.file.getName().getURI(); - // rooturi + // root URI extraData[outputIndex++] = data.file.getName().getRootURI(); - // childrens files - extraData[outputIndex++] = Long.valueOf(data.file.getChildren().length); + // nr of child files + extraData[outputIndex++] = (long) data.file.getChildren().length; // See if we need to add the row number to the row... - if (meta.includeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) { - extraData[outputIndex++] = Long.valueOf(data.rownr); + if (meta.isIncludeRowNumber() && !Utils.isEmpty(meta.getRowNumberField())) { + extraData[outputIndex] = data.rowNumber; } - data.rownr++; + data.rowNumber++; // Add row data - outputRow = RowDataUtil.addRowData(outputRow, data.totalpreviousfields, extraData); + outputRow = RowDataUtil.addRowData(outputRow, data.totalPreviousFields, extraData); // Send row putRow(data.outputRowMeta, outputRow); - // recurse until an output row has been created for each matched file - if (data.filenr < data.filessize) { - return processRow(); - } - if (meta.getRowLimit() > 0 - && data.rownr >= meta.getRowLimit()) { // limit has been reached: stop now. + && data.rowNumber >= meta.getRowLimit()) { // limit has been reached: stop now. setOutputDone(); - return false; + return true; } } } catch (Exception e) { throw new HopTransformException(e); } + return false; + } - data.filenr++; + private void initialize() throws HopException { + if (meta.isFolderNameDynamic()) { + data.inputRowMeta = getInputRowMeta(); + data.outputRowMeta = data.inputRowMeta.clone(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - if (checkFeedback(getLinesInput()) && log.isBasic()) { - logBasic(BaseMessages.getString(PKG, "GetSubFolders.Log.NrLine", "" + getLinesInput())); - } + // Get total previous fields + data.totalPreviousFields = data.inputRowMeta.size(); - return true; + // Check is filename field is provided + if (Utils.isEmpty(meta.getDynamicFolderNameField())) { + logError(BaseMessages.getString(PKG, "GetSubFolders.Log.NoField")); + throw new HopException(BaseMessages.getString(PKG, "GetSubFolders.Log.NoField")); + } + + // cache the position of the field + if (data.indexOfFolderNameField < 0) { + String realDynamicFolderName = resolve(meta.getDynamicFolderNameField()); + data.indexOfFolderNameField = data.inputRowMeta.indexOfValue(realDynamicFolderName); + if (data.indexOfFolderNameField < 0) { + // The field is unreachable ! + logError( + BaseMessages.getString(PKG, "GetSubFolders.Log.ErrorFindingField") + + "[" + + realDynamicFolderName + + "]"); + throw new HopException( + BaseMessages.getString( + PKG, "GetSubFolders.Exception.CouldnotFindField", realDynamicFolderName)); + } + } + } else { + // Create the output row meta-data + data.outputRowMeta = new RowMeta(); + meta.getFields( + data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); // get the + // metadata + // populated + + data.files = meta.getFolderList(this); + data.filesCount = data.files.nrOfFiles(); + handleMissingFiles(); + } + data.nrTransformFields = data.outputRowMeta.size(); } private void handleMissingFiles() throws HopException { List nonExistantFiles = data.files.getNonExistentFiles(); - if (nonExistantFiles.size() != 0) { + if (!nonExistantFiles.isEmpty()) { String message = FileInputList.getRequiredFilesDescription(nonExistantFiles); logError(BaseMessages.getString(PKG, "GetSubFolders.Error.MissingFiles", message)); throw new HopException( @@ -236,7 +223,7 @@ private void handleMissingFiles() throws HopException { } List nonAccessibleFiles = data.files.getNonAccessibleFiles(); - if (nonAccessibleFiles.size() != 0) { + if (!nonAccessibleFiles.isEmpty()) { String message = FileInputList.getRequiredFilesDescription(nonAccessibleFiles); logError(BaseMessages.getString(PKG, "GetSubFolders.Error.NoAccessibleFiles", message)); throw new HopException( @@ -246,16 +233,14 @@ private void handleMissingFiles() throws HopException { @Override public boolean init() { - if (super.init()) { try { - data.filessize = 0; - data.rownr = 1L; - data.filenr = 0; - data.totalpreviousfields = 0; + data.filesCount = 0; + data.rowNumber = 1L; + data.fileIndex = 0; + data.totalPreviousFields = 0; } catch (Exception e) { - logError("Error initializing transform: " + e.toString()); - logError(Const.getStackTracker(e)); + logError("Error initializing transform: ", e); return false; } diff --git a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersData.java b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersData.java index abc3d2c293f..5ae1dc128f0 100644 --- a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersData.java +++ b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersData.java @@ -33,33 +33,33 @@ public class GetSubFoldersData extends BaseTransformData implements ITransformDa public boolean isLastFile; - public int filenr; + public int fileIndex; - public int filessize; + public int filesCount; public FileObject file; - public long rownr; + public long rowNumber; - public int totalpreviousfields; + public int totalPreviousFields; - public int indexOfFoldernameField; + public int indexOfFolderNameField; public IRowMeta inputRowMeta; - public Object[] readrow; + public Object[] inputRow; public int nrTransformFields; public GetSubFoldersData() { super(); previousRow = null; - filenr = 0; - filessize = 0; + fileIndex = 0; + filesCount = 0; file = null; - totalpreviousfields = 0; - indexOfFoldernameField = -1; - readrow = null; + totalPreviousFields = 0; + indexOfFolderNameField = -1; + inputRow = null; nrTransformFields = 0; } } diff --git a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java index c376465461f..cba3461d1dd 100644 --- a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java +++ b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java @@ -25,7 +25,6 @@ import org.apache.hop.pipeline.Pipeline; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.PipelinePreviewFactory; -import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.ITransformDialog; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; @@ -42,8 +41,6 @@ import org.eclipse.swt.SWT; import org.eclipse.swt.custom.CTabFolder; import org.eclipse.swt.custom.CTabItem; -import org.eclipse.swt.events.FocusEvent; -import org.eclipse.swt.events.FocusListener; import org.eclipse.swt.events.ModifyListener; import org.eclipse.swt.events.SelectionAdapter; import org.eclipse.swt.events.SelectionEvent; @@ -55,46 +52,51 @@ import org.eclipse.swt.widgets.Composite; import org.eclipse.swt.widgets.Group; import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Listener; import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; import org.eclipse.swt.widgets.Text; public class GetSubFoldersDialog extends BaseTransformDialog implements ITransformDialog { private static final Class PKG = GetSubFoldersMeta.class; // For Translator - private Label wlFoldername; + public static final String YES = BaseMessages.getString(PKG, "System.Combo.Yes"); + public static final String NO = BaseMessages.getString(PKG, "System.Combo.No"); - private Button wbbFoldername; // Browse: add directory + private Label wlFolderName; - private Button wbdFoldername; // Delete + private Button wbbFolderName; // Browse: add directory - private Button wbeFoldername; // Edit + private Button wbdFolderName; // Delete - private Button wbaFoldername; // Add or change + private Button wbeFolderName; // Edit - private TextVar wFoldername; + private Button wbaFolderName; // Add or change - private Label wlFoldernameList; + private TextVar wFolderName; - private TableView wFoldernameList; + private Label wlFolderNameList; + + private TableView wFolderNameList; private final GetSubFoldersMeta input; private Button wFolderField; private Label wlFilenameField; - private ComboVar wFoldernameField; + private ComboVar wFolderNameField; private Label wlLimit; private Text wLimit; - private Button wInclRownum; + private Button wInclRowNumber; - private Label wlInclRownumField; - private TextVar wInclRownumField; + private Label wlInclRowNumberField; + private TextVar wInclRowNumberField; public GetSubFoldersDialog( Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { - super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); + super(parent, variables, (GetSubFoldersMeta) in, pipelineMeta, sname); input = (GetSubFoldersMeta) in; } @@ -117,7 +119,7 @@ public String open() { shell.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.DialogTitle")); int middle = props.getMiddlePct(); - int margin = props.getMargin(); + int margin = PropsUi.getMargin(); // Buttons at the bottom wOk = new Button(shell, SWT.PUSH); @@ -214,41 +216,36 @@ public void widgetSelected(SelectionEvent arg0) { wlFilenameField.setText( BaseMessages.getString(PKG, "GetSubFoldersDialog.wlFilenameField.Label")); PropsUi.setLook(wlFilenameField); - FormData fdlFoldernameField = new FormData(); - fdlFoldernameField.left = new FormAttachment(0, -margin); - fdlFoldernameField.top = new FormAttachment(wFolderField, margin); - fdlFoldernameField.right = new FormAttachment(middle, -2 * margin); - wlFilenameField.setLayoutData(fdlFoldernameField); - - wFoldernameField = new ComboVar(variables, wOriginFolders, SWT.BORDER | SWT.READ_ONLY); - wFoldernameField.setEditable(true); - PropsUi.setLook(wFoldernameField); - wFoldernameField.addModifyListener(lsMod); - FormData fdFoldernameField = new FormData(); - fdFoldernameField.left = new FormAttachment(middle, -margin); - fdFoldernameField.top = new FormAttachment(wFolderField, margin); - fdFoldernameField.right = new FormAttachment(100, -margin); - wFoldernameField.setLayoutData(fdFoldernameField); - wFoldernameField.setEnabled(false); - wFoldernameField.addFocusListener( - new FocusListener() { - @Override - public void focusLost(FocusEvent e) {} - - @Override - public void focusGained(FocusEvent e) { - Cursor busy = new Cursor(shell.getDisplay(), SWT.CURSOR_WAIT); - shell.setCursor(busy); - BaseTransformDialog.getFieldsFromPrevious( - variables, wFoldernameField, pipelineMeta, transformMeta); - shell.setCursor(null); - busy.dispose(); - } + FormData fdlFolderNameField = new FormData(); + fdlFolderNameField.left = new FormAttachment(0, -margin); + fdlFolderNameField.top = new FormAttachment(wFolderField, margin); + fdlFolderNameField.right = new FormAttachment(middle, -2 * margin); + wlFilenameField.setLayoutData(fdlFolderNameField); + + wFolderNameField = new ComboVar(variables, wOriginFolders, SWT.BORDER | SWT.READ_ONLY); + wFolderNameField.setEditable(true); + PropsUi.setLook(wFolderNameField); + wFolderNameField.addModifyListener(lsMod); + FormData fdFolderNameField = new FormData(); + fdFolderNameField.left = new FormAttachment(middle, -margin); + fdFolderNameField.top = new FormAttachment(wFolderField, margin); + fdFolderNameField.right = new FormAttachment(100, -margin); + wFolderNameField.setLayoutData(fdFolderNameField); + wFolderNameField.setEnabled(false); + wFolderNameField.addListener( + SWT.FocusIn, + e -> { + Cursor busy = new Cursor(shell.getDisplay(), SWT.CURSOR_WAIT); + shell.setCursor(busy); + BaseTransformDialog.getFieldsFromPrevious( + variables, wFolderNameField, pipelineMeta, transformMeta); + shell.setCursor(null); + busy.dispose(); }); FormData fdOriginFolders = new FormData(); fdOriginFolders.left = new FormAttachment(0, margin); - fdOriginFolders.top = new FormAttachment(wFoldernameList, margin); + fdOriginFolders.top = new FormAttachment(wFolderNameList, margin); fdOriginFolders.right = new FormAttachment(100, -margin); wOriginFolders.setLayoutData(fdOriginFolders); @@ -256,109 +253,110 @@ public void focusGained(FocusEvent e) { // / END OF Origin files GROUP // /////////////////////////////////////////////////////////// - // Foldername line - wlFoldername = new Label(wFolderComp, SWT.RIGHT); - wlFoldername.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.Filename.Label")); - PropsUi.setLook(wlFoldername); - FormData fdlFoldername = new FormData(); - fdlFoldername.left = new FormAttachment(0, 0); - fdlFoldername.top = new FormAttachment(wOriginFolders, margin); - fdlFoldername.right = new FormAttachment(middle, -margin); - wlFoldername.setLayoutData(fdlFoldername); - - wbbFoldername = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbbFoldername); - wbbFoldername.setText(BaseMessages.getString(PKG, "System.Button.Browse")); - wbbFoldername.setToolTipText( + // FolderName line + wlFolderName = new Label(wFolderComp, SWT.RIGHT); + wlFolderName.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.Filename.Label")); + PropsUi.setLook(wlFolderName); + FormData fdlFolderName = new FormData(); + fdlFolderName.left = new FormAttachment(0, 0); + fdlFolderName.top = new FormAttachment(wOriginFolders, margin); + fdlFolderName.right = new FormAttachment(middle, -margin); + wlFolderName.setLayoutData(fdlFolderName); + + wbbFolderName = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbbFolderName); + wbbFolderName.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + wbbFolderName.setToolTipText( BaseMessages.getString(PKG, "System.Tooltip.BrowseForFileOrDirAndAdd")); - FormData fdbFoldername = new FormData(); - fdbFoldername.right = new FormAttachment(100, 0); - fdbFoldername.top = new FormAttachment(wOriginFolders, margin); - wbbFoldername.setLayoutData(fdbFoldername); - - wbaFoldername = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbaFoldername); - wbaFoldername.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.FoldernameAdd.Button")); - wbaFoldername.setToolTipText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.FoldernameAdd.Tooltip")); - FormData fdbaFoldername = new FormData(); - fdbaFoldername.right = new FormAttachment(wbbFoldername, -margin); - fdbaFoldername.top = new FormAttachment(wOriginFolders, margin); - wbaFoldername.setLayoutData(fdbaFoldername); - - wFoldername = new TextVar(variables, wFolderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wFoldername); - wFoldername.addModifyListener(lsMod); - FormData fdFoldername = new FormData(); - fdFoldername.left = new FormAttachment(middle, 0); - fdFoldername.right = new FormAttachment(wbaFoldername, -margin); - fdFoldername.top = new FormAttachment(wOriginFolders, margin); - wFoldername.setLayoutData(fdFoldername); + FormData fdbFolderName = new FormData(); + fdbFolderName.right = new FormAttachment(100, 0); + fdbFolderName.top = new FormAttachment(wOriginFolders, margin); + wbbFolderName.setLayoutData(fdbFolderName); + + wbaFolderName = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbaFolderName); + wbaFolderName.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.FolderNameAdd.Button")); + wbaFolderName.setToolTipText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.FolderNameAdd.Tooltip")); + FormData fdbaFolderName = new FormData(); + fdbaFolderName.right = new FormAttachment(wbbFolderName, -margin); + fdbaFolderName.top = new FormAttachment(wOriginFolders, margin); + wbaFolderName.setLayoutData(fdbaFolderName); + + wFolderName = new TextVar(variables, wFolderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wFolderName); + wFolderName.addModifyListener(lsMod); + FormData fdFolderName = new FormData(); + fdFolderName.left = new FormAttachment(middle, 0); + fdFolderName.right = new FormAttachment(wbaFolderName, -margin); + fdFolderName.top = new FormAttachment(wOriginFolders, margin); + wFolderName.setLayoutData(fdFolderName); // Filename list line - wlFoldernameList = new Label(wFolderComp, SWT.RIGHT); - wlFoldernameList.setText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.FoldernameList.Label")); - PropsUi.setLook(wlFoldernameList); - FormData fdlFoldernameList = new FormData(); - fdlFoldernameList.left = new FormAttachment(0, 0); - fdlFoldernameList.top = new FormAttachment(wFoldername, 2 * margin); - fdlFoldernameList.right = new FormAttachment(middle, -margin); - wlFoldernameList.setLayoutData(fdlFoldernameList); + wlFolderNameList = new Label(wFolderComp, SWT.RIGHT); + wlFolderNameList.setText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.FolderNameList.Label")); + PropsUi.setLook(wlFolderNameList); + FormData fdlFolderNameList = new FormData(); + fdlFolderNameList.left = new FormAttachment(0, 0); + fdlFolderNameList.top = new FormAttachment(wFolderName, 2 * margin); + fdlFolderNameList.right = new FormAttachment(middle, -margin); + wlFolderNameList.setLayoutData(fdlFolderNameList); // Buttons to the right of the screen... - wbdFoldername = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbdFoldername); - wbdFoldername.setText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.FoldernameDelete.Button")); - wbdFoldername.setToolTipText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.FoldernameDelete.Tooltip")); - FormData fdbdFoldername = new FormData(); - fdbdFoldername.right = new FormAttachment(100, 0); - fdbdFoldername.top = new FormAttachment(wlFoldernameList, 0, SWT.TOP); - wbdFoldername.setLayoutData(fdbdFoldername); - - wbeFoldername = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); - PropsUi.setLook(wbeFoldername); - wbeFoldername.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.FilenameEdit.Button")); - wbeFoldername.setToolTipText( + wbdFolderName = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbdFolderName); + wbdFolderName.setText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.FolderNameDelete.Button")); + wbdFolderName.setToolTipText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.FolderNameDelete.Tooltip")); + FormData fdbdFolderName = new FormData(); + fdbdFolderName.right = new FormAttachment(100, 0); + fdbdFolderName.top = new FormAttachment(wlFolderNameList, 0, SWT.TOP); + wbdFolderName.setLayoutData(fdbdFolderName); + + wbeFolderName = new Button(wFolderComp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbeFolderName); + wbeFolderName.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.FilenameEdit.Button")); + wbeFolderName.setToolTipText( BaseMessages.getString(PKG, "GetSubFoldersDialog.FilenameEdit.Tooltip")); - FormData fdbeFoldername = new FormData(); - fdbeFoldername.right = new FormAttachment(100, 0); - fdbeFoldername.left = new FormAttachment(wbdFoldername, 0, SWT.LEFT); - fdbeFoldername.top = new FormAttachment(wbdFoldername, margin); - wbeFoldername.setLayoutData(fdbeFoldername); - - ColumnInfo[] colinfo = new ColumnInfo[2]; - colinfo[0] = + FormData fdbeFolderName = new FormData(); + fdbeFolderName.right = new FormAttachment(100, 0); + fdbeFolderName.left = new FormAttachment(wbdFolderName, 0, SWT.LEFT); + fdbeFolderName.top = new FormAttachment(wbdFolderName, margin); + wbeFolderName.setLayoutData(fdbeFolderName); + + ColumnInfo[] columns = new ColumnInfo[2]; + columns[0] = new ColumnInfo( BaseMessages.getString(PKG, "GetSubFoldersDialog.FileDirColumn.Column"), ColumnInfo.COLUMN_TYPE_TEXT, false); - colinfo[0].setUsingVariables(true); - colinfo[1] = + columns[0].setUsingVariables(true); + columns[1] = new ColumnInfo( BaseMessages.getString(PKG, "GetSubFoldersDialog.Required.Column"), ColumnInfo.COLUMN_TYPE_CCOMBO, - GetSubFoldersMeta.RequiredFoldersDesc); - colinfo[1].setToolTip(BaseMessages.getString(PKG, "GetSubFoldersDialog.Required.Tooltip")); + YES, + NO); + columns[1].setToolTip(BaseMessages.getString(PKG, "GetSubFoldersDialog.Required.Tooltip")); - wFoldernameList = + wFolderNameList = new TableView( variables, wFolderComp, SWT.FULL_SELECTION | SWT.SINGLE | SWT.BORDER, - colinfo, - colinfo.length, + columns, + input.getFiles().size(), lsMod, props); - PropsUi.setLook(wFoldernameList); - FormData fdFoldernameList = new FormData(); - fdFoldernameList.left = new FormAttachment(middle, 0); - fdFoldernameList.right = new FormAttachment(wbdFoldername, -margin); - fdFoldernameList.top = new FormAttachment(wlFoldernameList, 0, SWT.TOP); - fdFoldernameList.bottom = new FormAttachment(100, -margin); - wFoldernameList.setLayoutData(fdFoldernameList); + PropsUi.setLook(wFolderNameList); + FormData fdFolderNameList = new FormData(); + fdFolderNameList.left = new FormAttachment(middle, 0); + fdFolderNameList.right = new FormAttachment(wbdFolderName, -margin); + fdFolderNameList.top = new FormAttachment(wlFolderNameList, 0, SWT.TOP); + fdFolderNameList.bottom = new FormAttachment(100, -margin); + wFolderNameList.setLayoutData(fdFolderNameList); FormData fdFolderComp = new FormData(); fdFolderComp.left = new FormAttachment(0, 0); @@ -391,10 +389,10 @@ public void focusGained(FocusEvent e) { Composite wSettingsComp = new Composite(wTabFolder, SWT.NONE); PropsUi.setLook(wSettingsComp); - FormLayout filesettingLayout = new FormLayout(); - filesettingLayout.marginWidth = 3; - filesettingLayout.marginHeight = 3; - wSettingsComp.setLayout(fileLayout); + FormLayout fileSettingLayout = new FormLayout(); + fileSettingLayout.marginWidth = 3; + fileSettingLayout.marginHeight = 3; + wSettingsComp.setLayout(fileSettingLayout); // ///////////////////////////////// // START OF Additional Fields GROUP @@ -410,23 +408,23 @@ public void focusGained(FocusEvent e) { additionalgroupLayout.marginHeight = 10; wAdditionalGroup.setLayout(additionalgroupLayout); - Label wlInclRownum = new Label(wAdditionalGroup, SWT.RIGHT); - wlInclRownum.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRownum.Label")); - PropsUi.setLook(wlInclRownum); - FormData fdlInclRownum = new FormData(); - fdlInclRownum.left = new FormAttachment(0, 0); - fdlInclRownum.top = new FormAttachment(0, 2 * margin); - fdlInclRownum.right = new FormAttachment(middle, -margin); - wlInclRownum.setLayoutData(fdlInclRownum); - wInclRownum = new Button(wAdditionalGroup, SWT.CHECK); - PropsUi.setLook(wInclRownum); - wInclRownum.setToolTipText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRownum.Tooltip")); - FormData fdRownum = new FormData(); - fdRownum.left = new FormAttachment(middle, 0); - fdRownum.top = new FormAttachment(wlInclRownum, 0, SWT.CENTER); - wInclRownum.setLayoutData(fdRownum); - SelectionAdapter linclRownum = + Label wlInclRowNumber = new Label(wAdditionalGroup, SWT.RIGHT); + wlInclRowNumber.setText(BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRowNumber.Label")); + PropsUi.setLook(wlInclRowNumber); + FormData fdlInclRowNumber = new FormData(); + fdlInclRowNumber.left = new FormAttachment(0, 0); + fdlInclRowNumber.top = new FormAttachment(0, 2 * margin); + fdlInclRowNumber.right = new FormAttachment(middle, -margin); + wlInclRowNumber.setLayoutData(fdlInclRowNumber); + wInclRowNumber = new Button(wAdditionalGroup, SWT.CHECK); + PropsUi.setLook(wInclRowNumber); + wInclRowNumber.setToolTipText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRowNumber.Tooltip")); + FormData fdRowNumber = new FormData(); + fdRowNumber.left = new FormAttachment(middle, 0); + fdRowNumber.top = new FormAttachment(wlInclRowNumber, 0, SWT.CENTER); + wInclRowNumber.setLayoutData(fdRowNumber); + SelectionAdapter linclRowNumber = new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent arg0) { @@ -434,24 +432,25 @@ public void widgetSelected(SelectionEvent arg0) { input.setChanged(); } }; - wInclRownum.addSelectionListener(linclRownum); - - wlInclRownumField = new Label(wAdditionalGroup, SWT.RIGHT); - wlInclRownumField.setText( - BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRownumField.Label")); - PropsUi.setLook(wlInclRownumField); - FormData fdlInclRownumField = new FormData(); - fdlInclRownumField.left = new FormAttachment(wInclRownum, margin); - fdlInclRownumField.top = new FormAttachment(0, 2 * margin); - wlInclRownumField.setLayoutData(fdlInclRownumField); - wInclRownumField = new TextVar(variables, wAdditionalGroup, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wInclRownumField); - wInclRownumField.addModifyListener(lsMod); - FormData fdInclRownumField = new FormData(); - fdInclRownumField.left = new FormAttachment(wlInclRownumField, margin); - fdInclRownumField.top = new FormAttachment(0, 2 * margin); - fdInclRownumField.right = new FormAttachment(100, 0); - wInclRownumField.setLayoutData(fdInclRownumField); + wInclRowNumber.addSelectionListener(linclRowNumber); + + wlInclRowNumberField = new Label(wAdditionalGroup, SWT.RIGHT); + wlInclRowNumberField.setText( + BaseMessages.getString(PKG, "GetSubFoldersDialog.InclRowNumberField.Label")); + PropsUi.setLook(wlInclRowNumberField); + FormData fdlInclRowNumberField = new FormData(); + fdlInclRowNumberField.left = new FormAttachment(wInclRowNumber, margin); + fdlInclRowNumberField.top = new FormAttachment(0, 2 * margin); + wlInclRowNumberField.setLayoutData(fdlInclRowNumberField); + wInclRowNumberField = + new TextVar(variables, wAdditionalGroup, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wInclRowNumberField); + wInclRowNumberField.addModifyListener(lsMod); + FormData fdInclRowNumberField = new FormData(); + fdInclRowNumberField.left = new FormAttachment(wlInclRowNumberField, margin); + fdInclRowNumberField.top = new FormAttachment(0, 2 * margin); + fdInclRowNumberField.right = new FormAttachment(100, 0); + wInclRowNumberField.setLayoutData(fdInclRowNumberField); FormData fdAdditionalGroup = new FormData(); fdAdditionalGroup.left = new FormAttachment(0, margin); @@ -495,46 +494,43 @@ public void widgetSelected(SelectionEvent arg0) { // /////////////////////////////////////////////////////////// // Add the file to the list of files... - SelectionAdapter selA = - new SelectionAdapter() { - @Override - public void widgetSelected(SelectionEvent arg0) { - wFoldernameList.add(new String[] {wFoldername.getText()}); - wFoldername.setText(""); - wFoldernameList.removeEmptyRows(); - wFoldernameList.setRowNums(); - wFoldernameList.optWidth(true); - } + Listener selA = + e -> { + wFolderNameList.add(wFolderName.getText()); + wFolderName.setText(""); + wFolderNameList.removeEmptyRows(); + wFolderNameList.setRowNums(); + wFolderNameList.optWidth(true); }; - wbaFoldername.addSelectionListener(selA); - wFoldername.addSelectionListener(selA); + wbaFolderName.addListener(SWT.Selection, selA); + wFolderName.addListener(SWT.Selection, selA); // Delete files from the list of files... - wbdFoldername.addListener( + wbdFolderName.addListener( SWT.Selection, e -> { - int[] idx = wFoldernameList.getSelectionIndices(); - wFoldernameList.remove(idx); - wFoldernameList.removeEmptyRows(); - wFoldernameList.setRowNums(); + int[] idx = wFolderNameList.getSelectionIndices(); + wFolderNameList.remove(idx); + wFolderNameList.removeEmptyRows(); + wFolderNameList.setRowNums(); }); // Edit the selected file & remove from the list... - wbeFoldername.addListener( + wbeFolderName.addListener( SWT.Selection, e -> { - int idx = wFoldernameList.getSelectionIndex(); + int idx = wFolderNameList.getSelectionIndex(); if (idx >= 0) { - String[] string = wFoldernameList.getItem(idx); - wFoldername.setText(string[0]); - wFoldernameList.remove(idx); + String[] string = wFolderNameList.getItem(idx); + wFolderName.setText(string[0]); + wFolderNameList.remove(idx); } - wFoldernameList.removeEmptyRows(); - wFoldernameList.setRowNums(); + wFolderNameList.removeEmptyRows(); + wFolderNameList.setRowNums(); }); - wbbFoldername.addListener( - SWT.Selection, e -> BaseDialog.presentDirectoryDialog(shell, wFoldername, variables)); + wbbFolderName.addListener( + SWT.Selection, e -> BaseDialog.presentDirectoryDialog(shell, wFolderName, variables)); wTabFolder.setSelection(0); @@ -549,26 +545,23 @@ public void widgetSelected(SelectionEvent arg0) { } private void activateIncludeRowNum() { - wlInclRownumField.setEnabled(wInclRownum.getSelection()); - wInclRownumField.setEnabled(wInclRownum.getSelection()); + wlInclRowNumberField.setEnabled(wInclRowNumber.getSelection()); + wInclRowNumberField.setEnabled(wInclRowNumber.getSelection()); } private void activateFileField() { - if (wFolderField.getSelection()) { - wLimit.setText("0"); - } wlFilenameField.setEnabled(wFolderField.getSelection()); - wFoldernameField.setEnabled(wFolderField.getSelection()); - - wlFoldername.setEnabled(!wFolderField.getSelection()); - wbbFoldername.setEnabled(!wFolderField.getSelection()); - wbaFoldername.setEnabled(!wFolderField.getSelection()); - wFoldername.setEnabled(!wFolderField.getSelection()); - wlFoldernameList.setEnabled(!wFolderField.getSelection()); - wbdFoldername.setEnabled(!wFolderField.getSelection()); - wbeFoldername.setEnabled(!wFolderField.getSelection()); - wlFoldernameList.setEnabled(!wFolderField.getSelection()); - wFoldernameList.setEnabled(!wFolderField.getSelection()); + wFolderNameField.setEnabled(wFolderField.getSelection()); + + wlFolderName.setEnabled(!wFolderField.getSelection()); + wbbFolderName.setEnabled(!wFolderField.getSelection()); + wbaFolderName.setEnabled(!wFolderField.getSelection()); + wFolderName.setEnabled(!wFolderField.getSelection()); + wlFolderNameList.setEnabled(!wFolderField.getSelection()); + wbdFolderName.setEnabled(!wFolderField.getSelection()); + wbeFolderName.setEnabled(!wFolderField.getSelection()); + wlFolderNameList.setEnabled(!wFolderField.getSelection()); + wFolderNameList.setEnabled(!wFolderField.getSelection()); wPreview.setEnabled(!wFolderField.getSelection()); wlLimit.setEnabled(!wFolderField.getSelection()); wLimit.setEnabled(!wFolderField.getSelection()); @@ -580,30 +573,27 @@ private void activateFileField() { * @param meta The TextFileInputMeta object to obtain the data from. */ public void getData(GetSubFoldersMeta meta) { - final GetSubFoldersMeta in = meta; - - if (in.getFolderName() != null) { - wFoldernameList.removeAll(); - for (int i = 0; i < in.getFolderName().length; i++) { - wFoldernameList.add( - new String[] { - in.getFolderName()[i], in.getRequiredFilesDesc(in.getFolderRequired()[i]) - }); - } - wFoldernameList.removeEmptyRows(); - wFoldernameList.setRowNums(); - wFoldernameList.optWidth(true); - - wInclRownum.setSelection(in.includeRowNumber()); - wFolderField.setSelection(in.isFoldernameDynamic()); - if (in.getRowNumberField() != null) { - wInclRownumField.setText(in.getRowNumberField()); - } - if (in.getDynamicFoldernameField() != null) { - wFoldernameField.setText(in.getDynamicFoldernameField()); - } - wLimit.setText("" + in.getRowLimit()); + for (int i = 0; i < meta.getFiles().size(); i++) { + GetSubFoldersMeta.GSFile file = meta.getFiles().get(i); + TableItem item = wFolderNameList.table.getItem(i); + item.setText(1, Const.NVL(file.getName(), "")); + item.setText( + 2, + file.isRequired() + ? BaseMessages.getString(PKG, "System.Combo.Yes") + : BaseMessages.getString(PKG, "System.Combo.No")); } + wFolderNameList.optimizeTableView(); + + wInclRowNumber.setSelection(meta.isIncludeRowNumber()); + wFolderField.setSelection(meta.isFolderNameDynamic()); + if (meta.getRowNumberField() != null) { + wInclRowNumberField.setText(meta.getRowNumberField()); + } + if (meta.getDynamicFolderNameField() != null) { + wFolderNameField.setText(meta.getDynamicFolderNameField()); + } + wLimit.setText("" + meta.getRowLimit()); wTransformName.selectAll(); wTransformName.setFocus(); @@ -626,16 +616,18 @@ private void ok() { private void getInfo(GetSubFoldersMeta in) { transformName = wTransformName.getText(); // return value - int nrfiles = wFoldernameList.getItemCount(); - in.allocate(nrfiles); - - in.setFolderName(wFoldernameList.getItems(0)); - in.setFolderRequired(wFoldernameList.getItems(1)); + in.getFiles().clear(); + for (TableItem item : wFolderNameList.getNonEmptyItems()) { + GetSubFoldersMeta.GSFile file = new GetSubFoldersMeta.GSFile(); + file.setName(item.getText(1)); + file.setRequired(YES.equalsIgnoreCase(item.getText(2))); + in.getFiles().add(file); + } - in.setIncludeRowNumber(wInclRownum.getSelection()); - in.setDynamicFoldernameField(wFoldernameField.getText()); - in.setFolderField(wFolderField.getSelection()); - in.setRowNumberField(wInclRownumField.getText()); + in.setIncludeRowNumber(wInclRowNumber.getSelection()); + in.setDynamicFolderNameField(wFolderNameField.getText()); + in.setFolderNameDynamic(wFolderField.getSelection()); + in.setRowNumberField(wInclRowNumberField.getText()); in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); } diff --git a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMeta.java b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMeta.java index 5a93d1ff8dd..f68e6f1175a 100644 --- a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMeta.java +++ b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMeta.java @@ -19,12 +19,9 @@ import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.CheckResult; -import org.apache.hop.core.Const; import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.fileinput.FileInputList; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; @@ -35,16 +32,16 @@ import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; -import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransformMeta; import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.resource.IResourceNaming; import org.apache.hop.resource.ResourceDefinition; -import org.w3c.dom.Node; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -59,174 +56,56 @@ public class GetSubFoldersMeta extends BaseTransformMeta { private static final Class PKG = GetSubFoldersMeta.class; // For Translator - public static final String[] RequiredFoldersDesc = - new String[] { - BaseMessages.getString(PKG, "System.Combo.No"), - BaseMessages.getString(PKG, "System.Combo.Yes") - }; - public static final String[] RequiredFoldersCode = new String[] {"N", "Y"}; - - public static final String NO = "N"; - - /** Array of filenames */ - private String[] folderName; - - /** Array of boolean values as string, indicating if a file is required. */ - private String[] folderRequired; + /** The files/folders to get subfolders for */ + @HopMetadataProperty(key = "file") + private List files; /** Flag indicating that a row number field should be included in the output */ + @HopMetadataProperty(key = "rownum") private boolean includeRowNumber; /** The name of the field in the output containing the row number */ + @HopMetadataProperty(key = "rownum_field") private String rowNumberField; /** The name of the field in the output containing the foldername */ - private String dynamicFoldernameField; + @HopMetadataProperty(key = "foldername_field") + private String dynamicFolderNameField; /** folder name from previous fields */ - private boolean isFoldernameDynamic; + @HopMetadataProperty(key = "foldername_dynamic") + private boolean folderNameDynamic; /** The maximum number or lines to read */ + @HopMetadataProperty(key = "limit") private long rowLimit; public GetSubFoldersMeta() { - super(); // allocate BaseTransformMeta - } - - public String getRequiredFilesDesc(String tt) { - if (Utils.isEmpty(tt)) { - return RequiredFoldersDesc[0]; - } - if (tt.equalsIgnoreCase(RequiredFoldersCode[1])) { - return RequiredFoldersDesc[1]; - } else { - return RequiredFoldersDesc[0]; - } - } - - /** @return Returns the rowNumberField. */ - public String getRowNumberField() { - return rowNumberField; - } - - /** @param dynamicFoldernameField The dynamic foldername field to set. */ - public void setDynamicFoldernameField(String dynamicFoldernameField) { - this.dynamicFoldernameField = dynamicFoldernameField; - } - - /** @param rowNumberField The rowNumberField to set. */ - public void setRowNumberField(String rowNumberField) { - this.rowNumberField = rowNumberField; - } - - /** @return Returns the dynamic folder field (from previous transforms) */ - public String getDynamicFoldernameField() { - return dynamicFoldernameField; - } - - /** @return Returns the includeRowNumber. */ - public boolean includeRowNumber() { - return includeRowNumber; - } - - /** @return Returns the dynamic foldername flag. */ - public boolean isFoldernameDynamic() { - return isFoldernameDynamic; - } - - /** @param isFoldernameDynamic The isFoldernameDynamic to set. */ - public void setFolderField(boolean isFoldernameDynamic) { - this.isFoldernameDynamic = isFoldernameDynamic; - } - - /** @param includeRowNumber The includeRowNumber to set. */ - public void setIncludeRowNumber(boolean includeRowNumber) { - this.includeRowNumber = includeRowNumber; - } - - /** @return Returns the folderRequired. */ - public String[] getFolderRequired() { - return folderRequired; - } - - public String getRequiredFoldersCode(String tt) { - if (tt == null) { - return RequiredFoldersCode[0]; - } - if (tt.equals(RequiredFoldersDesc[1])) { - return RequiredFoldersCode[1]; - } else { - return RequiredFoldersCode[0]; - } - } - - /** @param folderRequiredin The folderRequired to set. */ - public void setFolderRequired(String[] folderRequiredin) { - this.folderRequired = new String[folderRequiredin.length]; - for (int i = 0; i < folderRequiredin.length; i++) { - this.folderRequired[i] = getRequiredFoldersCode(folderRequiredin[i]); - } - } - - /** @return Returns the folderName. */ - public String[] getFolderName() { - return folderName; - } - - /** @param folderName The folderName to set. */ - public void setFolderName(String[] folderName) { - this.folderName = folderName; - } - - /** @return Returns the rowLimit. */ - public long getRowLimit() { - return rowLimit; - } - - /** @param rowLimit The rowLimit to set. */ - public void setRowLimit(long rowLimit) { - this.rowLimit = rowLimit; + super(); + files = new ArrayList<>(); } - @Override - public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider) - throws HopXmlException { - readData(transformNode); + public GetSubFoldersMeta(GetSubFoldersMeta m) { + this(); + this.includeRowNumber = m.includeRowNumber; + this.rowNumberField = m.rowNumberField; + this.dynamicFolderNameField = m.dynamicFolderNameField; + this.folderNameDynamic = m.folderNameDynamic; + this.rowLimit = m.rowLimit; + m.files.forEach(f -> this.files.add(new GSFile(f))); } @Override - public Object clone() { - GetSubFoldersMeta retval = (GetSubFoldersMeta) super.clone(); - - int nrfiles = folderName.length; - - retval.allocate(nrfiles); - - System.arraycopy(folderName, 0, retval.folderName, 0, nrfiles); - System.arraycopy(folderRequired, 0, retval.folderRequired, 0, nrfiles); - - return retval; - } - - public void allocate(int nrfiles) { - folderName = new String[nrfiles]; - folderRequired = new String[nrfiles]; + public GetSubFoldersMeta clone() { + return new GetSubFoldersMeta(this); } @Override public void setDefault() { - int nrfiles = 0; - isFoldernameDynamic = false; + folderNameDynamic = false; includeRowNumber = false; rowNumberField = ""; - dynamicFoldernameField = ""; - - allocate(nrfiles); - - for (int i = 0; i < nrfiles; i++) { - folderName[i] = "folderName" + (i + 1); - folderRequired[i] = NO; - } + dynamicFolderNameField = ""; } @Override @@ -236,65 +115,64 @@ public void getFields( IRowMeta[] info, TransformMeta nextTransform, IVariables variables, - IHopMetadataProvider metadataProvider) - throws HopTransformException { + IHopMetadataProvider metadataProvider) { // the folderName - IValueMeta folderName = new ValueMetaString("folderName"); - folderName.setLength(500); - folderName.setPrecision(-1); - folderName.setOrigin(name); - row.addValueMeta(folderName); + IValueMeta folderNameValueMeta = new ValueMetaString("folderName"); + folderNameValueMeta.setLength(500); + folderNameValueMeta.setPrecision(-1); + folderNameValueMeta.setOrigin(name); + row.addValueMeta(folderNameValueMeta); // the short folderName - IValueMeta shortFolderName = new ValueMetaString("short_folderName"); - shortFolderName.setLength(500); - shortFolderName.setPrecision(-1); - shortFolderName.setOrigin(name); - row.addValueMeta(shortFolderName); + IValueMeta shortFolderNameValueMeta = new ValueMetaString("short_folderName"); + shortFolderNameValueMeta.setLength(500); + shortFolderNameValueMeta.setPrecision(-1); + shortFolderNameValueMeta.setOrigin(name); + row.addValueMeta(shortFolderNameValueMeta); // the path - IValueMeta path = new ValueMetaString("path"); - path.setLength(500); - path.setPrecision(-1); - path.setOrigin(name); - row.addValueMeta(path); - - // the ishidden - IValueMeta ishidden = new ValueMetaBoolean("ishidden"); - ishidden.setOrigin(name); - row.addValueMeta(ishidden); - - // the isreadable - IValueMeta isreadable = new ValueMetaBoolean("isreadable"); - isreadable.setOrigin(name); - row.addValueMeta(isreadable); - - // the iswriteable - IValueMeta iswriteable = new ValueMetaBoolean("iswriteable"); - iswriteable.setOrigin(name); - row.addValueMeta(iswriteable); - - // the lastmodifiedtime - IValueMeta lastmodifiedtime = new ValueMetaDate("lastmodifiedtime"); - lastmodifiedtime.setOrigin(name); - row.addValueMeta(lastmodifiedtime); + IValueMeta pathValueMeta = new ValueMetaString("path"); + pathValueMeta.setLength(500); + pathValueMeta.setPrecision(-1); + pathValueMeta.setOrigin(name); + row.addValueMeta(pathValueMeta); + + // the is hidden + IValueMeta isHiddenValueMeta = new ValueMetaBoolean("ishidden"); + isHiddenValueMeta.setOrigin(name); + row.addValueMeta(isHiddenValueMeta); + + // the is readable? + IValueMeta isReadableValueMeta = new ValueMetaBoolean("isreadable"); + isReadableValueMeta.setOrigin(name); + row.addValueMeta(isReadableValueMeta); + + // the is writeable + IValueMeta isWriteableValueMeta = new ValueMetaBoolean("iswriteable"); + isWriteableValueMeta.setOrigin(name); + row.addValueMeta(isWriteableValueMeta); + + // the last modified time + IValueMeta lastModifiedTimeValueMeta = new ValueMetaDate("lastmodifiedtime"); + lastModifiedTimeValueMeta.setOrigin(name); + row.addValueMeta(lastModifiedTimeValueMeta); // the uri - IValueMeta uri = new ValueMetaString("uri"); - uri.setOrigin(name); - row.addValueMeta(uri); + IValueMeta uriValueMeta = new ValueMetaString("uri"); + uriValueMeta.setOrigin(name); + row.addValueMeta(uriValueMeta); - // the rooturi - IValueMeta rooturi = new ValueMetaString("rooturi"); - rooturi.setOrigin(name); - row.addValueMeta(rooturi); + // the root uri + IValueMeta rootUriValueMeta = new ValueMetaString("rooturi"); + rootUriValueMeta.setOrigin(name); + row.addValueMeta(rootUriValueMeta); - // childrens - IValueMeta childrens = new ValueMetaInteger(variables.resolve("childrens")); - childrens.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0); - childrens.setOrigin(name); - row.addValueMeta(childrens); + // children + IValueMeta childrenValueMeta = new ValueMetaInteger(variables.resolve("childrens")); + childrenValueMeta.setLength(IValueMeta.DEFAULT_INTEGER_LENGTH, 0); + childrenValueMeta.setOrigin(name); + row.addValueMeta(childrenValueMeta); if (includeRowNumber) { IValueMeta v = new ValueMetaInteger(variables.resolve(rowNumberField)); @@ -304,57 +182,24 @@ public void getFields( } } - @Override - public String getXml() { - StringBuilder retval = new StringBuilder(300); - - retval.append(" ").append(XmlHandler.addTagValue("rownum", includeRowNumber)); - retval.append(" ").append(XmlHandler.addTagValue("foldername_dynamic", isFoldernameDynamic)); - retval.append(" ").append(XmlHandler.addTagValue("rownum_field", rowNumberField)); - retval - .append(" ") - .append(XmlHandler.addTagValue("foldername_field", dynamicFoldernameField)); - retval.append(" ").append(XmlHandler.addTagValue("limit", rowLimit)); - retval.append(" ").append(Const.CR); - - for (int i = 0; i < folderName.length; i++) { - retval.append(" ").append(XmlHandler.addTagValue("name", folderName[i])); - retval.append(" ").append(XmlHandler.addTagValue("file_required", folderRequired[i])); + public String[] getFilesNames() { + String[] names = new String[files.size()]; + for (int i = 0; i < names.length; i++) { + names[i] = files.get(i).getName(); } - retval.append(" ").append(Const.CR); - - return retval.toString(); + return names; } - private void readData(Node transformNode) throws HopXmlException { - try { - includeRowNumber = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "rownum")); - isFoldernameDynamic = - "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "foldername_dynamic")); - rowNumberField = XmlHandler.getTagValue(transformNode, "rownum_field"); - dynamicFoldernameField = XmlHandler.getTagValue(transformNode, "foldername_field"); - - // Is there a limit on the number of rows we process? - rowLimit = Const.toLong(XmlHandler.getTagValue(transformNode, "limit"), 0L); - - Node filenode = XmlHandler.getSubNode(transformNode, "file"); - int nrfiles = XmlHandler.countNodes(filenode, "name"); - - allocate(nrfiles); - - for (int i = 0; i < nrfiles; i++) { - Node folderNamenode = XmlHandler.getSubNodeByNr(filenode, "name", i); - Node folderRequirednode = XmlHandler.getSubNodeByNr(filenode, "file_required", i); - folderName[i] = XmlHandler.getNodeValue(folderNamenode); - folderRequired[i] = XmlHandler.getNodeValue(folderRequirednode); - } - } catch (Exception e) { - throw new HopXmlException("Unable to load transform info from XML", e); + public String[] getFilesRequired() { + String[] required = new String[files.size()]; + for (int i = 0; i < required.length; i++) { + required[i] = files.get(i).isRequired() ? "Y" : "N"; } + return required; } public FileInputList getFolderList(IVariables variables) { - return FileInputList.createFolderList(variables, folderName, folderRequired); + return FileInputList.createFolderList(variables, getFilesNames(), getFilesRequired()); } public FileInputList getDynamicFolderList( @@ -376,7 +221,7 @@ public void check( CheckResult cr; // See if we get input... - if (isFoldernameDynamic) { + if (folderNameDynamic) { if (input.length > 0) { cr = new CheckResult( @@ -392,7 +237,7 @@ public void check( } remarks.add(cr); - if (Utils.isEmpty(dynamicFoldernameField)) { + if (Utils.isEmpty(dynamicFolderNameField)) { cr = new CheckResult( ICheckResult.TYPE_RESULT_ERROR, @@ -452,8 +297,8 @@ public void check( * pray that the file is on a shared drive or something like that. * * @param variables the variable variables to use - * @param definitions - * @param iResourceNaming + * @param definitions The definitions to use + * @param iResourceNaming The naming method * @param metadataProvider the metadataProvider in which non-hop metadata could reside. * @return the filename of the exported resource */ @@ -469,10 +314,10 @@ public String exportResources( // So let's change the filename from relative to absolute by grabbing the file object... // In case the name of the file comes from previous transforms, forget about this! // - if (!isFoldernameDynamic) { - for (int i = 0; i < folderName.length; i++) { - FileObject fileObject = HopVfs.getFileObject(variables.resolve(folderName[i])); - folderName[i] = iResourceNaming.nameResource(fileObject, variables, true); + if (!folderNameDynamic) { + for (GSFile file : files) { + FileObject fileObject = HopVfs.getFileObject(variables.resolve(file.getName())); + file.setName(iResourceNaming.nameResource(fileObject, variables, true)); } } return null; @@ -480,4 +325,163 @@ public String exportResources( throw new HopException(e); } } + + public static final class GSFile { + @HopMetadataProperty(key = "name") + private String name; + + @HopMetadataProperty(key = "file_required") + private boolean required; + + public GSFile() {} + + public GSFile(GSFile f) { + this.name = f.name; + this.required = f.required; + } + + /** + * Gets name + * + * @return value of name + */ + public String getName() { + return name; + } + + /** + * Sets name + * + * @param name value of name + */ + public void setName(String name) { + this.name = name; + } + + /** + * Gets required + * + * @return value of required + */ + public boolean isRequired() { + return required; + } + + /** + * Sets required + * + * @param required value of required + */ + public void setRequired(boolean required) { + this.required = required; + } + } + + /** + * Gets files + * + * @return value of files + */ + public List getFiles() { + return files; + } + + /** + * Sets files + * + * @param files value of files + */ + public void setFiles(List files) { + this.files = files; + } + + /** + * Gets includeRowNumber + * + * @return value of includeRowNumber + */ + public boolean isIncludeRowNumber() { + return includeRowNumber; + } + + /** + * Sets includeRowNumber + * + * @param includeRowNumber value of includeRowNumber + */ + public void setIncludeRowNumber(boolean includeRowNumber) { + this.includeRowNumber = includeRowNumber; + } + + /** + * Gets rowNumberField + * + * @return value of rowNumberField + */ + public String getRowNumberField() { + return rowNumberField; + } + + /** + * Sets rowNumberField + * + * @param rowNumberField value of rowNumberField + */ + public void setRowNumberField(String rowNumberField) { + this.rowNumberField = rowNumberField; + } + + /** + * Gets dynamicFolderNameField + * + * @return value of dynamicFolderNameField + */ + public String getDynamicFolderNameField() { + return dynamicFolderNameField; + } + + /** + * Sets dynamicFolderNameField + * + * @param dynamicFolderNameField value of dynamicFolderNameField + */ + public void setDynamicFolderNameField(String dynamicFolderNameField) { + this.dynamicFolderNameField = dynamicFolderNameField; + } + + /** + * Gets folderNameDynamic + * + * @return value of folderNameDynamic + */ + public boolean isFolderNameDynamic() { + return folderNameDynamic; + } + + /** + * Sets folderNameDynamic + * + * @param folderNameDynamic value of folderNameDynamic + */ + public void setFolderNameDynamic(boolean folderNameDynamic) { + this.folderNameDynamic = folderNameDynamic; + } + + /** + * Gets rowLimit + * + * @return value of rowLimit + */ + public long getRowLimit() { + return rowLimit; + } + + /** + * Sets rowLimit + * + * @param rowLimit value of rowLimit + */ + public void setRowLimit(long rowLimit) { + this.rowLimit = rowLimit; + } } diff --git a/plugins/transforms/getsubfolders/src/test/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMetaTest.java b/plugins/transforms/getsubfolders/src/test/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMetaTest.java index 86fb3a93684..82df59e15c5 100644 --- a/plugins/transforms/getsubfolders/src/test/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMetaTest.java +++ b/plugins/transforms/getsubfolders/src/test/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersMetaTest.java @@ -17,141 +17,24 @@ package org.apache.hop.pipeline.transforms.getsubfolders; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.exception.HopTransformException; -import org.apache.hop.core.row.RowMeta; -import org.apache.hop.core.variables.Variables; -import org.apache.hop.junit.rules.RestoreHopEngineEnvironment; -import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; -import org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator; -import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator; -import org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator; -import org.junit.ClassRule; +import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; +import org.junit.Assert; import org.junit.Test; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - public class GetSubFoldersMetaTest { - @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); - - @Test - public void getFieldsTest() throws HopTransformException { - GetSubFoldersMeta transformMeta = new GetSubFoldersMeta(); - String transformName = UUID.randomUUID().toString(); - - RowMeta rowMeta = new RowMeta(); - transformMeta.getFields(rowMeta, transformName, null, null, new Variables(), null); - - assertFalse(transformMeta.includeRowNumber()); - assertEquals(10, rowMeta.size()); - assertEquals("folderName", rowMeta.getValueMeta(0).getName()); - assertEquals("short_folderName", rowMeta.getValueMeta(1).getName()); - assertEquals("path", rowMeta.getValueMeta(2).getName()); - assertEquals("ishidden", rowMeta.getValueMeta(3).getName()); - assertEquals("isreadable", rowMeta.getValueMeta(4).getName()); - assertEquals("iswriteable", rowMeta.getValueMeta(5).getName()); - assertEquals("lastmodifiedtime", rowMeta.getValueMeta(6).getName()); - assertEquals("uri", rowMeta.getValueMeta(7).getName()); - assertEquals("rooturi", rowMeta.getValueMeta(8).getName()); - assertEquals("childrens", rowMeta.getValueMeta(9).getName()); - - transformMeta.setIncludeRowNumber(true); - rowMeta = new RowMeta(); - transformMeta.getFields(rowMeta, transformName, null, null, new Variables(), null); - assertTrue(transformMeta.includeRowNumber()); - assertEquals(11, rowMeta.size()); - assertEquals("folderName", rowMeta.getValueMeta(0).getName()); - assertEquals("short_folderName", rowMeta.getValueMeta(1).getName()); - assertEquals("path", rowMeta.getValueMeta(2).getName()); - assertEquals("ishidden", rowMeta.getValueMeta(3).getName()); - assertEquals("isreadable", rowMeta.getValueMeta(4).getName()); - assertEquals("iswriteable", rowMeta.getValueMeta(5).getName()); - assertEquals("lastmodifiedtime", rowMeta.getValueMeta(6).getName()); - assertEquals("uri", rowMeta.getValueMeta(7).getName()); - assertEquals("rooturi", rowMeta.getValueMeta(8).getName()); - assertEquals("childrens", rowMeta.getValueMeta(9).getName()); - assertEquals(null, rowMeta.getValueMeta(10).getName()); - - transformMeta.setRowNumberField("MyRowNumber"); - rowMeta = new RowMeta(); - transformMeta.getFields(rowMeta, transformName, null, null, new Variables(), null); - assertEquals("MyRowNumber", transformMeta.getRowNumberField()); - assertEquals(11, rowMeta.size()); - assertEquals("MyRowNumber", rowMeta.getValueMeta(10).getName()); - } - @Test - public void loadSaveTest() throws HopException { - List attributes = - Arrays.asList( - "rownum", - "foldername_dynamic", - "rownum_field", - "foldername_field", - "limit", - "name", - "file_required"); - - Map getterMap = new HashMap<>(); - getterMap.put("rownum", "includeRowNumber"); - getterMap.put("foldername_dynamic", "isFoldernameDynamic"); - getterMap.put("foldername_field", "getDynamicFoldernameField"); - getterMap.put("rownum_field", "getRowNumberField"); - getterMap.put("limit", "getRowLimit"); - getterMap.put("name", "getFolderName"); - getterMap.put("file_required", "getFolderRequired"); - - Map setterMap = new HashMap<>(); - setterMap.put("rownum", "setIncludeRowNumber"); - setterMap.put("foldername_dynamic", "setFolderField"); - setterMap.put("foldername_field", "setDynamicFoldernameField"); - setterMap.put("rownum_field", "setRowNumberField"); - setterMap.put("limit", "setRowLimit"); - setterMap.put("name", "setFolderName"); - setterMap.put("file_required", "setFolderRequired"); - - Map> fieldLoadSaveValidatorAttributeMap = new HashMap<>(); - fieldLoadSaveValidatorAttributeMap.put( - "file_required", - new ArrayLoadSaveValidator<>(new FileRequiredFieldLoadSaveValidator(), 50)); - - Map> fieldLoadSaveValidatorTypeMap = new HashMap<>(); - fieldLoadSaveValidatorTypeMap.put( - String[].class.getCanonicalName(), - new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 50)); - - LoadSaveTester tester = - new LoadSaveTester( - GetSubFoldersMeta.class, - attributes, - getterMap, - setterMap, - fieldLoadSaveValidatorAttributeMap, - fieldLoadSaveValidatorTypeMap); - - tester.testSerialization(); - } - - public class FileRequiredFieldLoadSaveValidator implements IFieldLoadSaveValidator { - - @Override - public String getTestObject() { - return GetSubFoldersMeta.RequiredFoldersCode[ - new Random().nextInt(GetSubFoldersMeta.RequiredFoldersCode.length)]; - } - - @Override - public boolean validateTestObject(String testObject, Object actual) { - return testObject.equals(actual); - } + public void testSerialization() throws Exception { + GetSubFoldersMeta meta = + TransformSerializationTestUtil.testSerialization( + "/get-subfolder-names-transform.xml", GetSubFoldersMeta.class); + + Assert.assertTrue(meta.isIncludeRowNumber()); + Assert.assertEquals(1, meta.getFiles().size()); + Assert.assertEquals("${PROJECT_HOME}", meta.getFiles().get(0).getName()); + Assert.assertTrue(meta.getFiles().get(0).isRequired()); + Assert.assertEquals("rowNumber", meta.getRowNumberField()); + Assert.assertEquals(123L, meta.getRowLimit()); + Assert.assertEquals("inputField", meta.getDynamicFolderNameField()); + Assert.assertTrue(meta.isFolderNameDynamic()); } } diff --git a/plugins/transforms/getsubfolders/src/test/resources/get-subfolder-names-transform.xml b/plugins/transforms/getsubfolders/src/test/resources/get-subfolder-names-transform.xml new file mode 100644 index 00000000000..5200d98cf7e --- /dev/null +++ b/plugins/transforms/getsubfolders/src/test/resources/get-subfolder-names-transform.xml @@ -0,0 +1,29 @@ + + + + Y + Y + rowNumber + inputField + 123 + + ${PROJECT_HOME} + Y + +