From 3bd5192fdcf5e88e7632a19fed80e9b71d243d1d Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Mon, 25 May 2026 14:08:34 +0200 Subject: [PATCH 1/3] fix for required files and subfolders. fixes #7166 --- .../apache/hop/core/fileinput/InputFile.java | 2 +- .../transforms/file/BaseFileInputMeta.java | 3 + .../0100-json-input-include-subfolders.hpl | 170 +++++++++++++++ ...son-normalize-input-include-subfolders.hpl | 187 ++++++++++++++++ ...0-load-file-content-include-subfolders.hpl | 163 ++++++++++++++ ...100-text-file-input-include-subfolders.hpl | 202 ++++++++++++++++++ .../golden-json-input-include-subfolders.csv | 3 + ...son-normalize-input-include-subfolders.csv | 3 + ...n-load-file-content-include-subfolders.csv | 3 + ...den-text-file-input-include-subfolders.csv | 3 + .../root-events.json | 1 + .../file-input-include-subfolders/root.json | 1 + .../file-input-include-subfolders/root.txt | 2 + .../sub/nested-events.json | 1 + .../sub/nested.json | 1 + .../sub/nested.txt | 2 + ...ain-0100-file-input-include-subfolders.hwf | 89 ++++++++ .../golden-json-input-include-subfolders.json | 16 ++ ...on-normalize-input-include-subfolders.json | 24 +++ ...-load-file-content-include-subfolders.json | 16 ++ ...en-text-file-input-include-subfolders.json | 16 ++ ...00-json-input-include-subfolders UNIT.json | 28 +++ ...rmalize-input-include-subfolders UNIT.json | 33 +++ ...-file-content-include-subfolders UNIT.json | 28 +++ ...xt-file-input-include-subfolders UNIT.json | 28 +++ .../jsoninput/JsonInputMetaFileFlagsTest.java | 76 +++++++ .../loadfileinput/LoadFileInputDialog.java | 2 +- 27 files changed, 1101 insertions(+), 2 deletions(-) create mode 100644 integration-tests/transforms/0100-json-input-include-subfolders.hpl create mode 100644 integration-tests/transforms/0100-json-normalize-input-include-subfolders.hpl create mode 100644 integration-tests/transforms/0100-load-file-content-include-subfolders.hpl create mode 100644 integration-tests/transforms/0100-text-file-input-include-subfolders.hpl create mode 100644 integration-tests/transforms/datasets/golden-json-input-include-subfolders.csv create mode 100644 integration-tests/transforms/datasets/golden-json-normalize-input-include-subfolders.csv create mode 100644 integration-tests/transforms/datasets/golden-load-file-content-include-subfolders.csv create mode 100644 integration-tests/transforms/datasets/golden-text-file-input-include-subfolders.csv create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/root-events.json create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/root.json create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/root.txt create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/sub/nested-events.json create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/sub/nested.json create mode 100644 integration-tests/transforms/files/file-input-include-subfolders/sub/nested.txt create mode 100644 integration-tests/transforms/main-0100-file-input-include-subfolders.hwf create mode 100644 integration-tests/transforms/metadata/dataset/golden-json-input-include-subfolders.json create mode 100644 integration-tests/transforms/metadata/dataset/golden-json-normalize-input-include-subfolders.json create mode 100644 integration-tests/transforms/metadata/dataset/golden-load-file-content-include-subfolders.json create mode 100644 integration-tests/transforms/metadata/dataset/golden-text-file-input-include-subfolders.json create mode 100644 integration-tests/transforms/metadata/unit-test/0100-json-input-include-subfolders UNIT.json create mode 100644 integration-tests/transforms/metadata/unit-test/0100-json-normalize-input-include-subfolders UNIT.json create mode 100644 integration-tests/transforms/metadata/unit-test/0100-load-file-content-include-subfolders UNIT.json create mode 100644 integration-tests/transforms/metadata/unit-test/0100-text-file-input-include-subfolders UNIT.json create mode 100644 plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java diff --git a/core/src/main/java/org/apache/hop/core/fileinput/InputFile.java b/core/src/main/java/org/apache/hop/core/fileinput/InputFile.java index ed8c994c75..383225cc7f 100644 --- a/core/src/main/java/org/apache/hop/core/fileinput/InputFile.java +++ b/core/src/main/java/org/apache/hop/core/fileinput/InputFile.java @@ -87,7 +87,7 @@ public String getFileRequiredDesc() { } public String getIncludeSubFoldersDesc() { - if (fileRequired) { + if (includeSubFolders) { return BaseMessages.getString("System.Combo.Yes"); } else { return BaseMessages.getString("System.Combo.No"); diff --git a/engine/src/main/java/org/apache/hop/pipeline/transforms/file/BaseFileInputMeta.java b/engine/src/main/java/org/apache/hop/pipeline/transforms/file/BaseFileInputMeta.java index 843b8c7bb5..487f497d47 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/transforms/file/BaseFileInputMeta.java +++ b/engine/src/main/java/org/apache/hop/pipeline/transforms/file/BaseFileInputMeta.java @@ -131,10 +131,13 @@ public static void convertLegacyXml(List inputFiles, Node node) { XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode, "exclude_filemask", i)); String fileRequired = XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode, "file_required", i)); + String includeSubfolders = + XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode, "include_subfolders", i)); inputFile.setFileName(fileName); inputFile.setFileMask(fileMask); inputFile.setExcludeFileMask(fileExcludeMask); inputFile.setFileRequired(Const.toBoolean(fileRequired)); + inputFile.setIncludeSubFolders(Const.toBoolean(includeSubfolders)); inputFiles.add(inputFile); } } diff --git a/integration-tests/transforms/0100-json-input-include-subfolders.hpl b/integration-tests/transforms/0100-json-input-include-subfolders.hpl new file mode 100644 index 0000000000..e490335561 --- /dev/null +++ b/integration-tests/transforms/0100-json-input-include-subfolders.hpl @@ -0,0 +1,170 @@ + + + + + 0100-json-input-include-subfolders + Y + Issue 7166: JSON Input must honor include_subfolders when loading legacy file XML. + + + Normal + + + N + 1000 + 100 + - + 2026/05/25 12:00:00.000 + - + 2026/05/25 12:00:00.000 + + + + + + JSON input + Sort rows + Y + + + Sort rows + validate + Y + + + + JSON input + JsonInput + + Y + + 1 + + none + + + N + + N + N + N + N + N + Y + Y + Y + + + + ${PROJECT_HOME}/files/file-input-include-subfolders + ^[^/]+\.json$ + .*-events\.json$ + N + Y + + + + + id + $.id + String + + + + + -1 + -1 + none + N + + + 0 + N + N + + + + + + + + + + + + 80 + 64 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + N + + + id + Y + N + N + 0 + N + + + + + 224 + 64 + + + + validate + Dummy + + Y + + 1 + + none + + + + + 368 + 64 + + + + + + diff --git a/integration-tests/transforms/0100-json-normalize-input-include-subfolders.hpl b/integration-tests/transforms/0100-json-normalize-input-include-subfolders.hpl new file mode 100644 index 0000000000..1e3d971a0f --- /dev/null +++ b/integration-tests/transforms/0100-json-normalize-input-include-subfolders.hpl @@ -0,0 +1,187 @@ + + + + + 0100-json-normalize-input-include-subfolders + Y + Issue 7166: JSON Normalize Input must honor include_subfolders when loading legacy file XML. + + + Normal + + + N + 1000 + 100 + - + 2026/05/25 12:00:00.000 + - + 2026/05/25 12:00:00.000 + + + + + + JSON normalize input + Sort rows + Y + + + Sort rows + validate + Y + + + + JSON normalize input + JsonNormalizeInput + + Y + + 1 + + none + + + N + + N + + 0 + + N + N + N + N + Y + N + N + $.events[*] + . + -1 + STRINGIFY + STRINGIFY + Y + + + + + + + + + + + ${PROJECT_HOME}/files/file-input-include-subfolders + .*-events\.json$ + + N + Y + + + + + id + id + String + + none + -1 + -1 + + + + N + + + event + event + String + + none + -1 + -1 + + + + N + + + + + 80 + 64 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + N + + + id + Y + N + N + 0 + N + + + + + 224 + 64 + + + + validate + Dummy + + Y + + 1 + + none + + + + + 368 + 64 + + + + + + diff --git a/integration-tests/transforms/0100-load-file-content-include-subfolders.hpl b/integration-tests/transforms/0100-load-file-content-include-subfolders.hpl new file mode 100644 index 0000000000..9ae919cfae --- /dev/null +++ b/integration-tests/transforms/0100-load-file-content-include-subfolders.hpl @@ -0,0 +1,163 @@ + + + + + 0100-load-file-content-include-subfolders + Y + Issue 7166: Load file content must honor include_subfolders when loading legacy file XML. + + + Normal + + + N + 1000 + 100 + - + 2026/05/25 12:00:00.000 + - + 2026/05/25 12:00:00.000 + + + + + + Load file content + Sort rows + Y + + + Sort rows + validate + Y + + + + Load file content + LoadFileInput + + Y + + 1 + + none + + + N + + N + N + N + N + + UTF-8 + + ${PROJECT_HOME}/files/file-input-include-subfolders + .*\.txt$ + + N + Y + + + + file_size + size + Integer + + + + + -1 + -1 + none + N + + + 0 + N + + short_file + + + + + + + + + 80 + 64 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + N + + + short_file + Y + N + N + 0 + N + + + + + 224 + 64 + + + + validate + Dummy + + Y + + 1 + + none + + + + + 368 + 64 + + + + + + diff --git a/integration-tests/transforms/0100-text-file-input-include-subfolders.hpl b/integration-tests/transforms/0100-text-file-input-include-subfolders.hpl new file mode 100644 index 0000000000..b47bf8584f --- /dev/null +++ b/integration-tests/transforms/0100-text-file-input-include-subfolders.hpl @@ -0,0 +1,202 @@ + + + + + + N + 1000 + 100 + Normal + 0 + + 0100-text-file-input-include-subfolders + Y + Issue 7166: Text File Input must honor include_subfolders when loading legacy file XML. + + - + - + 2026/05/25 12:00:00.000 + 2026/05/25 12:00:00.000 + + + TextFileInput2 + Text file input + CSV + None + , + " + N + +
Y
+ N + 1 +
N
+ 1 + N + 1 + N + 80 + 0 + Y + N + + N + + N + mixed + UTF-8 + -1 + Y + en_US + Characters + + + + + N + + N + + + + + + + + + + + ${PROJECT_HOME}/files/file-input-include-subfolders + .*\.txt$ + + Y + Y + + + N + + N + + N + N + + + N + + + + + + + + + id + -1 + -1 + String + N + + none + -1 + + + + N + + + + + Y + 1 + + 80 + 64 + + + + none + + + + + + SortRows + Sort rows + + + id + Y + N + N + 0 + N + + + ${java.io.tmpdir} + out + 1000000 + + N + N + Y + 1 + + 224 + 64 + + + + none + + + + + + Dummy + validate + Y + 1 + + 368 + 64 + + + + none + + + + + + + Text file input + Sort rows + Y + + + Sort rows + validate + Y + + + + + + diff --git a/integration-tests/transforms/datasets/golden-json-input-include-subfolders.csv b/integration-tests/transforms/datasets/golden-json-input-include-subfolders.csv new file mode 100644 index 0000000000..3ef4a85f1c --- /dev/null +++ b/integration-tests/transforms/datasets/golden-json-input-include-subfolders.csv @@ -0,0 +1,3 @@ +id +nested +root diff --git a/integration-tests/transforms/datasets/golden-json-normalize-input-include-subfolders.csv b/integration-tests/transforms/datasets/golden-json-normalize-input-include-subfolders.csv new file mode 100644 index 0000000000..a41e823c19 --- /dev/null +++ b/integration-tests/transforms/datasets/golden-json-normalize-input-include-subfolders.csv @@ -0,0 +1,3 @@ +id,event +nested,e2 +root,e1 diff --git a/integration-tests/transforms/datasets/golden-load-file-content-include-subfolders.csv b/integration-tests/transforms/datasets/golden-load-file-content-include-subfolders.csv new file mode 100644 index 0000000000..fc5ad19ded --- /dev/null +++ b/integration-tests/transforms/datasets/golden-load-file-content-include-subfolders.csv @@ -0,0 +1,3 @@ +short_file +nested.txt +root.txt diff --git a/integration-tests/transforms/datasets/golden-text-file-input-include-subfolders.csv b/integration-tests/transforms/datasets/golden-text-file-input-include-subfolders.csv new file mode 100644 index 0000000000..3ef4a85f1c --- /dev/null +++ b/integration-tests/transforms/datasets/golden-text-file-input-include-subfolders.csv @@ -0,0 +1,3 @@ +id +nested +root diff --git a/integration-tests/transforms/files/file-input-include-subfolders/root-events.json b/integration-tests/transforms/files/file-input-include-subfolders/root-events.json new file mode 100644 index 0000000000..5bd5ae6325 --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/root-events.json @@ -0,0 +1 @@ +{"events": [{"id": "root", "event": "e1"}]} diff --git a/integration-tests/transforms/files/file-input-include-subfolders/root.json b/integration-tests/transforms/files/file-input-include-subfolders/root.json new file mode 100644 index 0000000000..465076acf8 --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/root.json @@ -0,0 +1 @@ +{"id": "root"} diff --git a/integration-tests/transforms/files/file-input-include-subfolders/root.txt b/integration-tests/transforms/files/file-input-include-subfolders/root.txt new file mode 100644 index 0000000000..1a9b4435b1 --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/root.txt @@ -0,0 +1,2 @@ +id +root diff --git a/integration-tests/transforms/files/file-input-include-subfolders/sub/nested-events.json b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested-events.json new file mode 100644 index 0000000000..3f552b4095 --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested-events.json @@ -0,0 +1 @@ +{"events": [{"id": "nested", "event": "e2"}]} diff --git a/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.json b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.json new file mode 100644 index 0000000000..c1cf6bdd6f --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.json @@ -0,0 +1 @@ +{"id": "nested"} diff --git a/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.txt b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.txt new file mode 100644 index 0000000000..22d4bed128 --- /dev/null +++ b/integration-tests/transforms/files/file-input-include-subfolders/sub/nested.txt @@ -0,0 +1,2 @@ +id +nested diff --git a/integration-tests/transforms/main-0100-file-input-include-subfolders.hwf b/integration-tests/transforms/main-0100-file-input-include-subfolders.hwf new file mode 100644 index 0000000000..a98ddc7369 --- /dev/null +++ b/integration-tests/transforms/main-0100-file-input-include-subfolders.hwf @@ -0,0 +1,89 @@ + + + + 0100-file-input-include-subfolders + Y + Issue 7166: verify file_required and include_subfolders for file-based input transforms. + + + - + 2026/05/25 12:00:00.000 + - + 2026/05/25 12:00:00.000 + + + + + Start + + SPECIAL + + 1 + N + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 80 + 64 + + + + 0100 file input include subfolders tests + + RunPipelineTests + + + + 0100-json-input-include-subfolders UNIT + + + 0100-text-file-input-include-subfolders UNIT + + + 0100-load-file-content-include-subfolders UNIT + + + 0100-json-normalize-input-include-subfolders UNIT + + + N + 320 + 64 + + + + + + Start + 0100 file input include subfolders tests + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-json-input-include-subfolders.json b/integration-tests/transforms/metadata/dataset/golden-json-input-include-subfolders.json new file mode 100644 index 0000000000..c304f13cf9 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-json-input-include-subfolders.json @@ -0,0 +1,16 @@ +{ + "base_filename": "golden-json-input-include-subfolders.csv", + "name": "golden-json-input-include-subfolders", + "description": "JSON Input with include_subfolders=Y must read root and nested json files.", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "id", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/dataset/golden-json-normalize-input-include-subfolders.json b/integration-tests/transforms/metadata/dataset/golden-json-normalize-input-include-subfolders.json new file mode 100644 index 0000000000..55cd58b439 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-json-normalize-input-include-subfolders.json @@ -0,0 +1,24 @@ +{ + "base_filename": "golden-json-normalize-input-include-subfolders.csv", + "name": "golden-json-normalize-input-include-subfolders", + "description": "JSON Normalize Input with include_subfolders=Y must read root and nested event json files.", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "id", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "event", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/dataset/golden-load-file-content-include-subfolders.json b/integration-tests/transforms/metadata/dataset/golden-load-file-content-include-subfolders.json new file mode 100644 index 0000000000..b1e91ff468 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-load-file-content-include-subfolders.json @@ -0,0 +1,16 @@ +{ + "base_filename": "golden-load-file-content-include-subfolders.csv", + "name": "golden-load-file-content-include-subfolders", + "description": "Load file content with include_subfolders=Y must read root and nested txt files.", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "short_file", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/dataset/golden-text-file-input-include-subfolders.json b/integration-tests/transforms/metadata/dataset/golden-text-file-input-include-subfolders.json new file mode 100644 index 0000000000..61a1867793 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-text-file-input-include-subfolders.json @@ -0,0 +1,16 @@ +{ + "base_filename": "golden-text-file-input-include-subfolders.csv", + "name": "golden-text-file-input-include-subfolders", + "description": "Text File Input with include_subfolders=Y must read root and nested txt files.", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "id", + "field_format": "" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/unit-test/0100-json-input-include-subfolders UNIT.json b/integration-tests/transforms/metadata/unit-test/0100-json-input-include-subfolders UNIT.json new file mode 100644 index 0000000000..6c2e015a2d --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0100-json-input-include-subfolders UNIT.json @@ -0,0 +1,28 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "Issue 7166: JSON Input include_subfolders", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "${HOP_UNIT_TESTS_FOLDER}", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + } + ], + "field_order": [ + "id" + ], + "data_set_name": "golden-json-input-include-subfolders", + "transform_name": "validate" + } + ], + "input_data_sets": [], + "name": "0100-json-input-include-subfolders UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0100-json-input-include-subfolders.hpl" +} diff --git a/integration-tests/transforms/metadata/unit-test/0100-json-normalize-input-include-subfolders UNIT.json b/integration-tests/transforms/metadata/unit-test/0100-json-normalize-input-include-subfolders UNIT.json new file mode 100644 index 0000000000..df3d3e6cc7 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0100-json-normalize-input-include-subfolders UNIT.json @@ -0,0 +1,33 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "Issue 7166: JSON Normalize Input include_subfolders", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "${HOP_UNIT_TESTS_FOLDER}", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "event", + "data_set_field": "event" + } + ], + "field_order": [ + "id", + "event" + ], + "data_set_name": "golden-json-normalize-input-include-subfolders", + "transform_name": "validate" + } + ], + "input_data_sets": [], + "name": "0100-json-normalize-input-include-subfolders UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0100-json-normalize-input-include-subfolders.hpl" +} diff --git a/integration-tests/transforms/metadata/unit-test/0100-load-file-content-include-subfolders UNIT.json b/integration-tests/transforms/metadata/unit-test/0100-load-file-content-include-subfolders UNIT.json new file mode 100644 index 0000000000..d46fa0e430 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0100-load-file-content-include-subfolders UNIT.json @@ -0,0 +1,28 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "Issue 7166: Load file content include_subfolders", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "${HOP_UNIT_TESTS_FOLDER}", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "short_file", + "data_set_field": "short_file" + } + ], + "field_order": [ + "short_file" + ], + "data_set_name": "golden-load-file-content-include-subfolders", + "transform_name": "validate" + } + ], + "input_data_sets": [], + "name": "0100-load-file-content-include-subfolders UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0100-load-file-content-include-subfolders.hpl" +} diff --git a/integration-tests/transforms/metadata/unit-test/0100-text-file-input-include-subfolders UNIT.json b/integration-tests/transforms/metadata/unit-test/0100-text-file-input-include-subfolders UNIT.json new file mode 100644 index 0000000000..1ea85752a2 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0100-text-file-input-include-subfolders UNIT.json @@ -0,0 +1,28 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "Issue 7166: Text File Input include_subfolders", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "${HOP_UNIT_TESTS_FOLDER}", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "id", + "data_set_field": "id" + } + ], + "field_order": [ + "id" + ], + "data_set_name": "golden-text-file-input-include-subfolders", + "transform_name": "validate" + } + ], + "input_data_sets": [], + "name": "0100-text-file-input-include-subfolders UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0100-text-file-input-include-subfolders.hpl" +} diff --git a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java new file mode 100644 index 0000000000..771eab1fa8 --- /dev/null +++ b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java @@ -0,0 +1,76 @@ +package org.apache.hop.pipeline.transforms.jsoninput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import org.apache.hop.core.fileinput.InputFile; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaJson; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaPlugin; +import org.apache.hop.core.row.value.ValueMetaPluginType; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class JsonInputMetaFileFlagsTest { + + @BeforeEach + void beforeEach() throws Exception { + PluginRegistry registry = PluginRegistry.getInstance(); + String[] classNames = { + ValueMetaString.class.getName(), + ValueMetaInteger.class.getName(), + ValueMetaDate.class.getName(), + ValueMetaNumber.class.getName(), + ValueMetaJson.class.getName() + }; + for (String className : classNames) { + registry.registerPluginClass(className, ValueMetaPluginType.class, ValueMetaPlugin.class); + } + } + + @Test + void legacyXmlLoadsFileRequiredAndIncludeSubfolders() throws Exception { + Path path = + Paths.get(Objects.requireNonNull(getClass().getResource("/json-input.xml")).toURI()); + JsonInputMeta meta = new JsonInputMeta(); + XmlMetadataUtil.deSerializeFromXml( + XmlHandler.loadXmlString(Files.readString(path), TransformMeta.XML_TAG), + JsonInputMeta.class, + meta, + new MemoryMetadataProvider()); + + assertEquals(2, meta.getFileInput().getInputFiles().size()); + InputFile first = meta.getFileInput().getInputFiles().get(0); + InputFile second = meta.getFileInput().getInputFiles().get(1); + assertFalse(first.isFileRequired()); + assertFalse(first.isIncludeSubFolders()); + assertTrue(second.isFileRequired()); + assertTrue(second.isIncludeSubFolders()); + } + + @Test + void includeSubFoldersDescReflectsIncludeSubFoldersField() { + InputFile file = new InputFile(); + file.setFileRequired(true); + file.setIncludeSubFolders(false); + assertTrue(file.getFileRequiredDesc().contains("Y") || file.getFileRequiredDesc().length() > 0); + assertFalse( + file.getIncludeSubFoldersDesc().equals(file.getFileRequiredDesc()), + "include subfolders description must not mirror file required"); + file.setIncludeSubFolders(true); + assertEquals(file.getFileRequiredDesc(), file.getIncludeSubFoldersDesc()); + } +} diff --git a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java index b72ebf38fa..8d8c484e60 100644 --- a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java +++ b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java @@ -1159,7 +1159,7 @@ public void getData(LoadFileInputMeta in) { inputFile.getFileMask(), inputFile.getExcludeFileMask(), inputFile.isFileRequired() ? CONST_COMBO_YES : CONST_COMBO_NO, - inputFile.isFileRequired() ? CONST_COMBO_YES : CONST_COMBO_NO); + inputFile.isIncludeSubFolders() ? CONST_COMBO_YES : CONST_COMBO_NO); } wFilenameList.optimizeTableView(); From 2f341f2bfa7d8b969f1507349840aac8006ccb9a Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Mon, 25 May 2026 14:13:37 +0200 Subject: [PATCH 2/3] added missing ASF header. fixes #7166 --- .../jsoninput/JsonInputMetaFileFlagsTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java index 771eab1fa8..b1cceddebc 100644 --- a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java +++ b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaFileFlagsTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hop.pipeline.transforms.jsoninput; import static org.junit.jupiter.api.Assertions.assertEquals; From adea47bb414b9599a07a50147415a65cf12d5f78 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Mon, 25 May 2026 16:06:05 +0200 Subject: [PATCH 3/3] fixed category for JSON Normalize Input transform. fixes #7170 --- .../transforms/jsonnormalize/JsonNormalizeInputMeta.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java index 54b3177a28..979f632226 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java @@ -57,7 +57,7 @@ name = "i18n::JsonNormalizeInput.name", description = "i18n::JsonNormalizeInput.description", keywords = "i18n::JsonNormalizeInputMeta.keyword", - categoryDescription = "i18n::JsonInput.category") + categoryDescription = "i18n:org.apache.hop.pipeline.transforms.jsoninput:JsonInput.category") @Getter @Setter public class JsonNormalizeInputMeta