From 85858f57007258bcc4d438322f9dfdb77d67d49a Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Fri, 8 May 2026 13:09:41 +0200 Subject: [PATCH 1/9] first implementation of advanced XML output --- .../transforms/advancedxmloutput.adoc | 113 +++ .../advancedxmloutput/AdvancedXmlOutput.java | 812 ++++++++++++++++++ .../AdvancedXmlOutputData.java | 82 ++ .../AdvancedXmlOutputDialog.java | 202 +++++ .../AdvancedXmlOutputMeta.java | 296 +++++++ .../AdvancedXmlOutputValidator.java | 108 +++ .../XmlFileOutputSupport.java | 214 +++++ .../xml/advancedxmloutput/XmlNode.java | 239 ++++++ .../transforms/xml/src/main/resources/AXO.svg | 38 + .../messages/messages_en_US.properties | 60 ++ .../AdvancedXmlOutputMetaTest.java | 127 +++ .../AdvancedXmlOutputTest.java | 253 ++++++ .../test/resources/advanced-xml-output.xml | 83 ++ 13 files changed, 2627 insertions(+) create mode 100644 docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputValidator.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java create mode 100644 plugins/transforms/xml/src/main/resources/AXO.svg create mode 100644 plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties create mode 100644 plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java create mode 100644 plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java create mode 100644 plugins/transforms/xml/src/test/resources/advanced-xml-output.xml diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc new file mode 100644 index 0000000000..d511979ae0 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc @@ -0,0 +1,113 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The Advanced XML Output transform writes rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. + += image:transforms/icons/AXO.svg[Advanced XML Output transform Icon, role="image-doc-icon"] Advanced XML Output + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The Advanced XML Output transform writes rows from any source to one or more XML files, using a hierarchical, user-defined XML tree. + +The XML tree is a recursive structure of elements, attributes and document-fragment nodes. Exactly one element in the tree must be marked as the row-*loop*: each input row produces one occurrence of that element with its full subtree. Optionally, ancestors of the loop can be marked as *group-by*: consecutive input rows that share the same group key are emitted under a single occurrence of the group element. + +This transform complements the simpler `XML Output` transform. `XML Output` is the right choice when you want a flat document of repeating rows; `Advanced XML Output` is the right choice when you need a deeper, custom-shaped XML structure (loops nested inside groups, attributes at any level, document fragments, namespaces). + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:cross.svg[Not Supported, 24] +!Flink! image:cross.svg[Not Supported, 24] +!Dataflow! image:cross.svg[Not Supported, 24] +!=== +|=== + +== Options + +[NOTE] +==== +The Phase-1 dialog only exposes the basic file options. The interactive XML tree designer is on the way; until it ships, configure the tree by editing the transform metadata directly (HopGui's "Show metadata" command, or by hand in your `.hpl` file). +==== + +=== File Tab + +[options="header"] +|=== +|Option|Description +|Transform name|Name of the transform. +|Filename|Base name of the output XML file (without extension). +|Extension|File extension (without the leading dot). Defaults to `xml`. +|Encoding|Character encoding for the output file. Defaults to UTF-8. +|Add filename to result|When enabled, the produced filename is added to the pipeline's result file list (only after at least one row is written). +|=== + +=== Content options + +The following advanced options are available on the metadata (and will be exposed in the dialog in the next phase): + +[options="header"] +|=== +|Option|Description +|`compactFile`|Suppress whitespace and EOL between elements; useful for byte-size-sensitive output. +|`blankLineAfterXmlDeclaration`|Add a blank line right after the `` declaration (default `true`, matching common formatters). +|`createEmptyElement`|Emit an open/close tag pair for an element that has no value and no children (default `true`). +|`createAttributeIfNull`|Emit an attribute even when its source value is `null`. +|`createAttributeIfUnmapped`|Emit an attribute that has no mapped field (using its default value). +|`trimValues`|Trim leading/trailing whitespace from emitted text values. +|`defaultDecimalSeparator`|Default decimal separator for numeric values; per-node settings still take precedence. +|`defaultGroupingSeparator`|Default grouping separator for numeric values; per-node settings still take precedence. +|`generateXsd`|When set and the tree declares at least one namespace, write a sibling `.xsd` next to the output file. +|`doctypeRootElement` / `doctypeSystemId` / `doctypePublicId`|Emit a `` declaration between the XML declaration and the root element. +|`xslStylesheetHref` / `xslStylesheetType`|Emit an `` processing instruction; defaults to `text/xsl` when type is empty. +|`splitEvery`|Maximum rows per file before splitting (`0` = no split). +|`zipped`|Wrap the output file in a zip archive. +|`doNotOpenNewFileInit`|Defer file creation until the first input row is received. +|`doNotCreateEmptyFile`|Delete the file at the end of the run if no rows were written. +|=== + +=== XML Tree + +The XML tree is a recursive `XmlNode` graph. Each node has the following properties: + +[options="header"] +|=== +|Property|Description +|`name`|Local name of the element or attribute. +|`namespace`|Optional XML namespace URI (treated as default namespace at that element). +|`kind`|`Element`, `Attribute`, or `DocumentFragment` (the latter parses the source field's value and inserts it as XML nodes rather than escaped text). +|`mappedField`|Input field whose value provides this node's content. For attributes and elements it sets the value; for nodes flagged `groupBy`, it identifies the group key. +|`defaultValue`|Static text used when `mappedField` is empty (or its value is `null`). +|`format`, `length`, `precision`, `currencySymbol`, `decimalSymbol`, `groupingSymbol`|Optional value-meta overrides used when converting the field value to a string. +|`forceCreate`|Output this node even when the value is `null`. +|`loop`|Marks this element as the row-loop element. Exactly one element in the tree must be flagged. +|`groupBy`|Marks this element as a group-by ancestor of the loop. Consecutive rows with equal `mappedField` values share a single occurrence. +|`children`|Recursive list of child `XmlNode`s. +|=== + +== Group-by behaviour + +For the group-by mechanism to collapse correctly, *the input rows must already be sorted by the group-by key(s)*. Use a Sort Rows transform upstream if needed. When the key changes, the open group element is closed and a new one is opened. + +== Memory profile + +The transform uses StAX streaming and only buffers the XML state of the currently-open path of group elements. A single very large group is therefore O(largest group) in memory rather than O(document). diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java new file mode 100644 index 0000000000..c125d0a5c0 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -0,0 +1,812 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.io.File; +import java.io.OutputStream; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.Const; +import org.apache.hop.core.ResultFile; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.io.CountingOutputStream; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** + * Runtime engine for the Advanced XML Output transform. + * + *

Walks the configured {@link XmlNode} tree once at first-row time, splitting it into a "prefix + * path" (root → loop's parent) with optional group-by ancestors, a "loop subtree" emitted for each + * input row, and a "suffix" of closing tags. Group-by ancestors collapse consecutive input rows + * that share the same group key into a single occurrence of the group element. Memory profile is + * O(largest group); the writer is StAX-streaming. + */ +public class AdvancedXmlOutput extends BaseTransform { + + private static final String EOL = "\n"; + private static final XMLOutputFactory XML_OUT_FACTORY = XMLOutputFactory.newInstance(); + private static final XMLInputFactory XML_IN_FACTORY = createSecureInputFactory(); + + private OutputStream outputStream; + + /** The currently-open path of {@link XmlNode}s above the loop, deepest first. */ + private final Deque openStack = new ArrayDeque<>(); + + public AdvancedXmlOutput( + TransformMeta transformMeta, + AdvancedXmlOutputMeta meta, + AdvancedXmlOutputData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + // --------------------------------------------------------------------------- + // BaseTransform overrides + // --------------------------------------------------------------------------- + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + data.splitnr = 0; + if (meta.getFileSupport().isDoNotOpenNewFileInit()) { + return true; + } + if (openNewFile()) { + return true; + } + logError("Couldn't open file " + meta.getFileSupport().getFileName()); + setErrors(1L); + stopAll(); + return false; + } + + @Override + public boolean processRow() throws HopException { + Object[] r = getRow(); + + if (first) { + first = false; + if (r == null) { + // No rows at all + if (data.fileOpen && meta.getFileSupport().isDoNotCreateEmptyFile()) { + discardEmptyFile(); + } else if (data.fileOpen) { + // Open file, then immediately close as empty (root + decl). + finalizeOpenPath(); + closeFile(); + } + setOutputDone(); + return false; + } + data.inputRowMeta = getInputRowMeta(); + resolveTreeStructure(); + } + + // Handle split BEFORE writing the row so the new file's prefix is fresh. + int splitEvery = meta.getFileSupport().getSplitEvery(); + if (r != null + && getLinesOutput() > 0 + && splitEvery > 0 + && (getLinesOutput() % splitEvery) == 0) { + finalizeOpenPath(); + closeFile(); + if (!openNewFile()) { + logError("Unable to open new file (split #" + data.splitnr + ")"); + setErrors(1); + return false; + } + } + + if (r == null) { + finalizeOpenPath(); + if (!data.rowsWrittenToCurrentFile && meta.getFileSupport().isDoNotCreateEmptyFile()) { + discardEmptyFile(); + } else { + closeFile(); + } + setOutputDone(); + return false; + } + + // Lazy open if the user asked us to defer file creation. + if (!data.fileOpen && meta.getFileSupport().isDoNotOpenNewFileInit()) { + if (!openNewFile()) { + logError("Couldn't open file " + meta.getFileSupport().getFileName()); + setErrors(1); + return false; + } + } + + try { + writeRowToTree(r); + } catch (Exception e) { + throw new HopException( + "Error writing XML row :" + e + Const.CR + "Row: " + getInputRowMeta().getString(r), e); + } + + putRow(getInputRowMeta(), r); + + if (checkFeedback(getLinesOutput()) && isBasic()) { + logBasic("linenr " + getLinesOutput()); + } + return true; + } + + @Override + public void dispose() { + try { + finalizeOpenPath(); + if (!data.rowsWrittenToCurrentFile && meta.getFileSupport().isDoNotCreateEmptyFile()) { + discardEmptyFile(); + } else { + closeFile(); + } + } catch (Exception e) { + logError("Error closing XML output", e); + } + super.dispose(); + } + + // --------------------------------------------------------------------------- + // Tree resolution / first-row setup + // --------------------------------------------------------------------------- + + /** + * Walk the tree once to locate the loop node, the path from root to the loop's parent, and the + * ordered list of group-by ancestors. Resolves field indices for every node that references an + * input field. + */ + private void resolveTreeStructure() throws HopException { + XmlNode root = meta.getRootNode(); + if (root == null) { + throw new HopException("XML tree is not configured."); + } + + // Validate + List errors = AdvancedXmlOutputValidator.validate(root, data.inputRowMeta); + if (!errors.isEmpty()) { + throw new HopException(String.join("; ", errors)); + } + + // Locate the (single) loop node and the path to it. + List pathToLoop = new ArrayList<>(); + if (!findPathToLoop(root, pathToLoop)) { + throw new HopException( + "Could not locate the loop element in the XML tree (mark exactly one node as loop)."); + } + + // pathToLoop = [root, ..., loop]. pathToLoopParent = [root, ..., loop's parent]. + data.loopNode = pathToLoop.get(pathToLoop.size() - 1); + data.pathToLoopParent = new ArrayList<>(pathToLoop.subList(0, pathToLoop.size() - 1)); + + // Group-by ancestors of the loop, in document order (top → bottom). + data.groupByPath = new ArrayList<>(); + for (XmlNode anc : data.pathToLoopParent) { + if (anc.isGroupBy()) { + data.groupByPath.add(anc); + } + } + + // Resolve field indices for every node with a mapped field, anywhere in the tree. + data.fieldIndex = new IdentityHashMap<>(); + resolveFieldIndices(root); + } + + private boolean findPathToLoop(XmlNode node, List path) { + path.add(node); + if (node.isLoop()) { + return true; + } + if (node.getChildren() != null) { + for (XmlNode child : node.getChildren()) { + if (findPathToLoop(child, path)) { + return true; + } + } + } + path.remove(path.size() - 1); + return false; + } + + private void resolveFieldIndices(XmlNode node) throws HopException { + if (!Utils.isEmpty(node.getMappedField())) { + int idx = data.inputRowMeta.indexOfValue(node.getMappedField()); + if (idx < 0) { + throw new HopException( + "Field [" + + node.getMappedField() + + "] not found in input row for node '" + + node.getName() + + "'."); + } + data.fieldIndex.put(node, idx); + // Apply formatting overrides to the value-meta (clone-safe; we work on input row meta clone) + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + if (vm != null) { + if (!Utils.isEmpty(node.getFormat())) { + vm.setConversionMask(node.getFormat()); + } + if (node.getLength() > 0 || node.getPrecision() > 0) { + vm.setLength(node.getLength(), node.getPrecision()); + } + if (!Utils.isEmpty(node.getCurrencySymbol())) { + vm.setCurrencySymbol(node.getCurrencySymbol()); + } + if (!Utils.isEmpty(node.getDecimalSymbol())) { + vm.setDecimalSymbol(node.getDecimalSymbol()); + } else if (!Utils.isEmpty(meta.getDefaultDecimalSeparator())) { + vm.setDecimalSymbol(meta.getDefaultDecimalSeparator()); + } + if (!Utils.isEmpty(node.getGroupingSymbol())) { + vm.setGroupingSymbol(node.getGroupingSymbol()); + } else if (!Utils.isEmpty(meta.getDefaultGroupingSeparator())) { + vm.setGroupingSymbol(meta.getDefaultGroupingSeparator()); + } + } + } + if (node.getChildren() != null) { + for (XmlNode c : node.getChildren()) { + resolveFieldIndices(c); + } + } + } + + // --------------------------------------------------------------------------- + // File handling + // --------------------------------------------------------------------------- + + private boolean openNewFile() { + data.writer = null; + try { + String physicalName = + meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, true); + String innerName = meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, false); + FileObject file = HopVfs.getFileObject(physicalName, variables); + data.currentFile = file; + data.currentFileName = physicalName; + + OutputStream fos = HopVfs.getOutputStream(file, false); + data.countingStream = new CountingOutputStream(fos); + if (meta.getFileSupport().isZipped()) { + data.zip = new ZipOutputStream(data.countingStream); + ZipEntry entry = new ZipEntry(new File(innerName).getName()); + entry.setComment("Compressed by Apache Hop"); + data.zip.putNextEntry(entry); + outputStream = data.zip; + } else { + outputStream = data.countingStream; + } + + String enc = Utils.isEmpty(meta.getEncoding()) ? Const.UTF_8 : meta.getEncoding(); + data.writer = XML_OUT_FACTORY.createXMLStreamWriter(outputStream, enc); + data.writer.writeStartDocument(enc, "1.0"); + if (!meta.isCompactFile() && meta.isBlankLineAfterXmlDeclaration()) { + data.writer.writeCharacters(EOL); + } + + writeOptionalDocType(); + writeOptionalXslPi(); + + data.fileOpen = true; + data.rowsWrittenToCurrentFile = false; + data.currentGroupKey = null; + openStack.clear(); + data.splitnr++; + return true; + } catch (Exception e) { + logError("Error opening new file: " + e); + return false; + } + } + + private void writeOptionalDocType() throws Exception { + String root = variables.resolve(meta.getDoctypeRootElement()); + String sys = variables.resolve(meta.getDoctypeSystemId()); + String pub = variables.resolve(meta.getDoctypePublicId()); + if (Utils.isEmpty(root)) { + return; + } + StringBuilder sb = new StringBuilder(""); + data.writer.writeDTD(sb.toString()); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + } + + private void writeOptionalXslPi() throws Exception { + String href = variables.resolve(meta.getXslStylesheetHref()); + if (Utils.isEmpty(href)) { + return; + } + String type = variables.resolve(meta.getXslStylesheetType()); + if (Utils.isEmpty(type)) { + type = "text/xsl"; + } + data.writer.writeProcessingInstruction( + "xml-stylesheet", "type=\"" + type + "\" href=\"" + href + "\""); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + } + + private void finalizeOpenPath() { + if (data.writer == null || !data.fileOpen) { + return; + } + try { + // Close any open path elements (deepest first). + while (!openStack.isEmpty()) { + data.writer.writeEndElement(); + openStack.pop(); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + } + } catch (Exception e) { + logError("Error finalizing XML path: " + e); + } + } + + private void closeFile() { + if (!data.fileOpen) { + return; + } + try { + data.writer.writeEndDocument(); + data.writer.close(); + + if (meta.getFileSupport().isZipped()) { + data.zip.closeEntry(); + data.zip.finish(); + } + if (data.countingStream != null) { + dataVolumeOut = + (dataVolumeOut != null ? dataVolumeOut : 0L) + data.countingStream.getCount(); + data.countingStream.close(); + } + } catch (Exception e) { + logError("Error closing XML output: " + e); + } finally { + data.fileOpen = false; + } + } + + /** Discards a file that was opened but received no rows. */ + private void discardEmptyFile() { + try { + if (data.writer != null) { + try { + data.writer.close(); + } catch (Exception ignore) { + // best effort + } + } + if (meta.getFileSupport().isZipped() && data.zip != null) { + try { + data.zip.close(); + } catch (Exception ignore) { + // best effort + } + } + if (data.countingStream != null) { + try { + data.countingStream.close(); + } catch (Exception ignore) { + // best effort + } + } + if (data.currentFile != null && data.currentFile.exists()) { + // We never registered the file as a result (registration is deferred to the + // first successful row write), so just delete it. + data.currentFile.delete(); + } + } catch (Exception e) { + logError("Error discarding empty XML file: " + e); + } finally { + data.fileOpen = false; + } + } + + // --------------------------------------------------------------------------- + // Row-level write + // --------------------------------------------------------------------------- + + private void writeRowToTree(Object[] r) throws Exception { + String[] newKey = computeGroupKey(r); + + if (data.currentGroupKey == null) { + // First row: open the entire path from root to loop's parent. + openPath(0, data.pathToLoopParent.size() - 1, r); + data.currentGroupKey = newKey; + } else { + int changedLevel = newKey.length; + for (int i = 0; i < newKey.length; i++) { + if (!Objects.equals(data.currentGroupKey[i], newKey[i])) { + changedLevel = i; + break; + } + } + if (changedLevel < newKey.length) { + // Close from end-of-path back down through groupByPath[changedLevel] (inclusive). + int closeDownToSize = pathIndexOf(data.groupByPath.get(changedLevel)); + while (openStack.size() > closeDownToSize) { + data.writer.writeEndElement(); + openStack.pop(); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + } + openPath(closeDownToSize, data.pathToLoopParent.size() - 1, r); + data.currentGroupKey = newKey; + } + } + + // Emit the loop subtree (one row → one occurrence of the loop element). + writeSubtree(data.loopNode, r); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + + incrementLinesOutput(); + if (!data.rowsWrittenToCurrentFile) { + // Defer result-file registration until we actually write something. + registerResultFile(); + } + data.rowsWrittenToCurrentFile = true; + } + + private void registerResultFile() { + if (!meta.getFileSupport().isAddToResultFilenames()) { + return; + } + if (data.currentFile == null) { + return; + } + ResultFile rf = + new ResultFile( + ResultFile.FILE_TYPE_GENERAL, + data.currentFile, + getPipelineMeta().getName(), + getTransformName()); + rf.setComment("File created by Advanced XML Output transform"); + addResultFile(rf); + } + + /** Returns the index of {@code node} within {@link AdvancedXmlOutputData#pathToLoopParent}. */ + private int pathIndexOf(XmlNode node) throws HopException { + for (int i = 0; i < data.pathToLoopParent.size(); i++) { + if (data.pathToLoopParent.get(i) == node) { + return i; + } + } + throw new HopException("Internal error: node not on path: " + node); + } + + /** Builds the current row's group-key tuple (one entry per group-by ancestor of the loop). */ + private String[] computeGroupKey(Object[] r) throws Exception { + String[] key = new String[data.groupByPath.size()]; + for (int i = 0; i < data.groupByPath.size(); i++) { + XmlNode g = data.groupByPath.get(i); + Integer idx = data.fieldIndex.get(g); + key[i] = idx == null ? "" : data.inputRowMeta.getValueMeta(idx).getString(r[idx]); + } + return key; + } + + /** + * Open path elements from index {@code from} (inclusive) up to {@code to} (inclusive). For each + * opened element we also emit attribute children, the element's own mapped value (if any), and + * any non-path static descendant subtrees, taking the values from the current row. + */ + private void openPath(int from, int to, Object[] r) throws Exception { + // Set of XmlNode identities that lie on the path from this node down to the loop. Children + // matching one of these must NOT be emitted as static content (they are handled by the next + // openPath/openLoop call). + Set onPath = onPathSet(); + + for (int i = from; i <= to; i++) { + XmlNode node = data.pathToLoopParent.get(i); + writeStartElementWithNamespace(node); + + // Attributes first (StAX requires attributes before character/element content). + writeAttributesOf(node, r); + + // Element's own mapped value, if any (text content before children). + writeOwnTextValue(node, r); + + // Static (non-on-path) child subtrees. + if (node.getChildren() != null) { + for (XmlNode c : node.getChildren()) { + if (c.getKind() == XmlNode.NodeKind.Attribute) { + continue; // already written + } + if (onPath.contains(c)) { + continue; // path will be opened next iteration / loop + } + writeSubtree(c, r); + } + } + + openStack.push(node); + if (!meta.isCompactFile()) { + data.writer.writeCharacters(EOL); + } + } + } + + private Set onPathSet() { + Set s = java.util.Collections.newSetFromMap(new IdentityHashMap<>()); + s.addAll(data.pathToLoopParent); + s.add(data.loopNode); + return s; + } + + /** Recursively writes a complete element/attribute subtree using the given row. */ + private void writeSubtree(XmlNode node, Object[] r) throws Exception { + switch (node.getKind()) { + case Attribute -> { + // Attribute writing is handled by the parent's writeAttributesOf(). + } + case DocumentFragment -> writeDocumentFragment(node, r); + case Element -> writeElement(node, r); + } + } + + private void writeElement(XmlNode node, Object[] r) throws Exception { + Integer idx = data.fieldIndex.get(node); + boolean hasMapping = idx != null; + String value = null; + if (hasMapping) { + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + Object data1 = r[idx]; + value = vm.isNull(data1) ? null : vm.getString(data1); + } else if (!Utils.isEmpty(node.getDefaultValue())) { + value = node.getDefaultValue(); + } + + boolean hasChildren = node.hasElementChildren(); + boolean hasAttributes = node.hasAttributeChildren(); + boolean shouldEmit = + hasChildren + || hasAttributes + || value != null + || node.isForceCreate() + || meta.isCreateEmptyElement(); + + if (!shouldEmit) { + return; + } + + writeStartElementWithNamespace(node); + writeAttributesOf(node, r); + if (value != null) { + data.writer.writeCharacters(applyTrim(value)); + } + if (node.getChildren() != null) { + for (XmlNode c : node.getChildren()) { + if (c.getKind() == XmlNode.NodeKind.Attribute) { + continue; + } + writeSubtree(c, r); + } + } + data.writer.writeEndElement(); + } + + private void writeStartElementWithNamespace(XmlNode node) throws Exception { + if (!Utils.isEmpty(node.getNamespace())) { + data.writer.writeStartElement(node.getNamespace(), node.getName()); + data.writer.writeDefaultNamespace(node.getNamespace()); + } else { + data.writer.writeStartElement(node.getName()); + } + } + + /** Writes any direct attribute children of {@code node} for the current row. */ + private void writeAttributesOf(XmlNode node, Object[] r) throws Exception { + if (node.getChildren() == null) { + return; + } + for (XmlNode c : node.getChildren()) { + if (c.getKind() != XmlNode.NodeKind.Attribute) { + continue; + } + Integer idx = data.fieldIndex.get(c); + String value = null; + if (idx != null) { + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + Object data1 = r[idx]; + value = vm.isNull(data1) ? null : vm.getString(data1); + } else if (!Utils.isEmpty(c.getDefaultValue())) { + value = c.getDefaultValue(); + } + + if (value == null) { + if (meta.isCreateAttributeIfNull() || c.isForceCreate()) { + data.writer.writeAttribute( + c.getName(), Utils.isEmpty(c.getDefaultValue()) ? "" : c.getDefaultValue()); + } + // else: skip silently + } else { + data.writer.writeAttribute(c.getName(), applyTrim(value)); + } + } + // Unmapped attributes that the user wants to emit anyway (rare; v1 covers via forceCreate). + if (meta.isCreateAttributeIfUnmapped() && node.getChildren() != null) { + for (XmlNode c : node.getChildren()) { + if (c.getKind() != XmlNode.NodeKind.Attribute) { + continue; + } + if (data.fieldIndex.containsKey(c)) { + continue; // already handled + } + if (Utils.isEmpty(c.getMappedField()) && !c.isForceCreate()) { + // forceCreate path is already covered above with default value + data.writer.writeAttribute( + c.getName(), Utils.isEmpty(c.getDefaultValue()) ? "" : c.getDefaultValue()); + } + } + } + } + + /** + * Writes the element's own text content (mapped field or default value), if applicable. For a + * group-by node, the mapped field is interpreted purely as a group key and is NOT written as the + * element's text content (use a child element / attribute to surface the value). + */ + private void writeOwnTextValue(XmlNode node, Object[] r) throws Exception { + if (node.isGroupBy()) { + // Group-by node: mappedField is the group key only. + if (!Utils.isEmpty(node.getDefaultValue())) { + data.writer.writeCharacters(applyTrim(node.getDefaultValue())); + } + return; + } + Integer idx = data.fieldIndex.get(node); + String value = null; + if (idx != null) { + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + Object data1 = r[idx]; + value = vm.isNull(data1) ? null : vm.getString(data1); + } else if (!Utils.isEmpty(node.getDefaultValue())) { + value = node.getDefaultValue(); + } + if (value != null) { + data.writer.writeCharacters(applyTrim(value)); + } + } + + /** + * Inserts the value of the mapped field as parsed XML nodes (rather than escaped text). The field + * value is expected to contain a well-formed XML fragment (or fragment with multiple top + * elements). On parse failure we log a warning and emit nothing. + */ + private void writeDocumentFragment(XmlNode node, Object[] r) throws Exception { + Integer idx = data.fieldIndex.get(node); + if (idx == null) { + return; + } + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + Object data1 = r[idx]; + if (vm.isNull(data1)) { + return; + } + String fragment = vm.getString(data1); + if (Utils.isEmpty(fragment)) { + return; + } + // Wrap so we can have multiple top-level nodes. + String wrapped = "" + fragment + ""; + try (Reader reader = new StringReader(wrapped)) { + XMLStreamReader xr = XML_IN_FACTORY.createXMLStreamReader(reader); + int depth = 0; + while (xr.hasNext()) { + int event = xr.next(); + switch (event) { + case XMLStreamConstants.START_ELEMENT -> { + depth++; + if (depth == 1 && "root".equals(xr.getLocalName())) { + break; // skip the wrapper itself + } + data.writer.writeStartElement(xr.getLocalName()); + for (int i = 0; i < xr.getAttributeCount(); i++) { + data.writer.writeAttribute(xr.getAttributeLocalName(i), xr.getAttributeValue(i)); + } + } + case XMLStreamConstants.END_ELEMENT -> { + if (depth == 1 && "root".equals(xr.getLocalName())) { + depth--; + break; + } + data.writer.writeEndElement(); + depth--; + } + case XMLStreamConstants.CHARACTERS, XMLStreamConstants.CDATA -> { + if (depth >= 2 || (depth == 1 && !"root".equals(xr.getLocalName()))) { + data.writer.writeCharacters(xr.getText()); + } + } + default -> { + // ignore comments, PIs, whitespace + } + } + } + xr.close(); + } catch (Exception e) { + logError( + "Could not parse XML document fragment from field '" + + node.getMappedField() + + "': " + + e.getMessage()); + } + } + + private String applyTrim(String value) { + return meta.isTrimValues() && value != null ? value.trim() : value; + } + + private static XMLInputFactory createSecureInputFactory() { + XMLInputFactory f = XMLInputFactory.newInstance(); + f.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + f.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return f; + } + + // --------------------------------------------------------------------------- + // Helpers exposed for tests + // --------------------------------------------------------------------------- + + /** Test hook: returns the current data object's writer (so unit tests can inject a mock). */ + protected XMLStreamWriter getWriter() { + return data == null ? null : data.writer; + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java new file mode 100644 index 0000000000..4930ddd12c --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.util.List; +import java.util.zip.ZipOutputStream; +import javax.xml.stream.XMLStreamWriter; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.io.CountingOutputStream; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +@SuppressWarnings("java:S1104") +public class AdvancedXmlOutputData extends BaseTransformData implements ITransformData { + + /** Increments each time a new (split) file is opened. */ + public int splitnr; + + /** Whether {@code openNewFile} ran successfully and the file is now open. */ + public boolean fileOpen; + + /** Whether the currently open file has had at least one row written to it. */ + public boolean rowsWrittenToCurrentFile; + + /** The last opened {@link FileObject} (kept around so we can delete on empty-file). */ + public FileObject currentFile; + + /** The "inner" filename when the file is zipped; equals the on-disk name otherwise. */ + public String currentFileName; + + /** Optional zip wrapper around the underlying output stream. */ + public ZipOutputStream zip; + + /** Bytes-written counter for metrics. */ + public CountingOutputStream countingStream; + + /** The StAX writer wrapping the (possibly zipped) output stream. */ + public XMLStreamWriter writer; + + /** Row meta of the input stream, captured on first row. */ + public IRowMeta inputRowMeta; + + /** Resolved field indices for every node that references an input field (by node identity). */ + public java.util.Map fieldIndex; + + /** Group-by ancestors (top→down) of the loop node, captured once at first-row time. */ + public List groupByPath; + + /** The single loop node, captured once at first-row time. */ + public XmlNode loopNode; + + /** Path from root (inclusive) to the loop node's parent (inclusive). */ + public List pathToLoopParent; + + /** + * Last group-key tuple written. Same length as {@link #groupByPath}; null = no group is open yet. + */ + public String[] currentGroupKey; + + public AdvancedXmlOutputData() { + super(); + this.splitnr = 0; + this.fileOpen = false; + this.rowsWrittenToCurrentFile = false; + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java new file mode 100644 index 0000000000..c91f1d22c5 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import org.apache.hop.core.Const; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; + +/** + * Phase-1 stub dialog for the Advanced XML Output transform. Exposes only the filename and encoding + * so the transform is editable in HopGui while the full tree designer (Phase 2) is implemented. The + * XML tree itself is configured programmatically or by editing the metadata directly until the + * designer ships. + */ +public class AdvancedXmlOutputDialog extends BaseTransformDialog { + + private static final Class PKG = AdvancedXmlOutputMeta.class; + + private final AdvancedXmlOutputMeta input; + + private TextVar wFilename; + private TextVar wExtension; + private TextVar wEncoding; + private Button wAddToResult; + + public AdvancedXmlOutputDialog( + Shell parent, + IVariables variables, + AdvancedXmlOutputMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + this.input = transformMeta; + } + + @Override + public String open() { + createShell(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.DialogTitle")); + + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + + ModifyListener lsMod = e -> input.setChanged(); + changed = input.hasChanged(); + + Composite wMain = new Composite(shell, SWT.NONE); + PropsUi.setLook(wMain); + FormLayout layout = new FormLayout(); + layout.marginWidth = 3; + layout.marginHeight = 3; + wMain.setLayout(layout); + + // Filename + Label wlFilename = new Label(wMain, SWT.RIGHT); + wlFilename.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Filename.Label")); + PropsUi.setLook(wlFilename); + FormData fdlFilename = new FormData(); + fdlFilename.left = new FormAttachment(0, 0); + fdlFilename.top = new FormAttachment(0, margin); + fdlFilename.right = new FormAttachment(middle, -margin); + wlFilename.setLayoutData(fdlFilename); + wFilename = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wFilename); + wFilename.addModifyListener(lsMod); + FormData fdFilename = new FormData(); + fdFilename.left = new FormAttachment(middle, 0); + fdFilename.top = new FormAttachment(0, margin); + fdFilename.right = new FormAttachment(100, 0); + wFilename.setLayoutData(fdFilename); + + // Extension + Label wlExt = new Label(wMain, SWT.RIGHT); + wlExt.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Extension.Label")); + PropsUi.setLook(wlExt); + FormData fdlExt = new FormData(); + fdlExt.left = new FormAttachment(0, 0); + fdlExt.top = new FormAttachment(wFilename, margin); + fdlExt.right = new FormAttachment(middle, -margin); + wlExt.setLayoutData(fdlExt); + wExtension = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wExtension); + wExtension.addModifyListener(lsMod); + FormData fdExt = new FormData(); + fdExt.left = new FormAttachment(middle, 0); + fdExt.top = new FormAttachment(wFilename, margin); + fdExt.right = new FormAttachment(100, 0); + wExtension.setLayoutData(fdExt); + + // Encoding + Label wlEnc = new Label(wMain, SWT.RIGHT); + wlEnc.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Encoding.Label")); + PropsUi.setLook(wlEnc); + FormData fdlEnc = new FormData(); + fdlEnc.left = new FormAttachment(0, 0); + fdlEnc.top = new FormAttachment(wExtension, margin); + fdlEnc.right = new FormAttachment(middle, -margin); + wlEnc.setLayoutData(fdlEnc); + wEncoding = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wEncoding); + wEncoding.addModifyListener(lsMod); + FormData fdEnc = new FormData(); + fdEnc.left = new FormAttachment(middle, 0); + fdEnc.top = new FormAttachment(wExtension, margin); + fdEnc.right = new FormAttachment(100, 0); + wEncoding.setLayoutData(fdEnc); + + // Add to result + Label wlAddToResult = new Label(wMain, SWT.RIGHT); + wlAddToResult.setText( + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.AddFileToResult.Label")); + PropsUi.setLook(wlAddToResult); + FormData fdlA = new FormData(); + fdlA.left = new FormAttachment(0, 0); + fdlA.top = new FormAttachment(wEncoding, margin); + fdlA.right = new FormAttachment(middle, -margin); + wlAddToResult.setLayoutData(fdlA); + wAddToResult = new Button(wMain, SWT.CHECK); + PropsUi.setLook(wAddToResult); + FormData fdA = new FormData(); + fdA.left = new FormAttachment(middle, 0); + fdA.top = new FormAttachment(wlAddToResult, 0, SWT.CENTER); + fdA.right = new FormAttachment(100, 0); + wAddToResult.setLayoutData(fdA); + + // Notice + Label wlNotice = new Label(wMain, SWT.WRAP); + wlNotice.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Notice.Label")); + PropsUi.setLook(wlNotice); + FormData fdN = new FormData(); + fdN.left = new FormAttachment(0, 0); + fdN.right = new FormAttachment(100, 0); + fdN.top = new FormAttachment(wAddToResult, margin * 3); + wlNotice.setLayoutData(fdN); + + FormData fdMain = new FormData(); + fdMain.left = new FormAttachment(0, 0); + fdMain.top = new FormAttachment(wSpacer, margin); + fdMain.right = new FormAttachment(100, 0); + fdMain.bottom = new FormAttachment(100, -50); + wMain.setLayoutData(fdMain); + + getData(); + input.setChanged(changed); + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + + return transformName; + } + + private void getData() { + wFilename.setText(Const.NVL(input.getFileSupport().getFileName(), "")); + wExtension.setText(Const.NVL(input.getFileSupport().getExtension(), "")); + wEncoding.setText(Const.NVL(input.getEncoding(), Const.UTF_8)); + wAddToResult.setSelection(input.getFileSupport().isAddToResultFilenames()); + } + + private void cancel() { + transformName = null; + input.setChanged(backupChanged); + dispose(); + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + transformName = wTransformName.getText(); + input.getFileSupport().setFileName(wFilename.getText()); + input.getFileSupport().setExtension(wExtension.getText()); + input.setEncoding(wEncoding.getText()); + input.getFileSupport().setAddToResultFilenames(wAddToResult.getSelection()); + dispose(); + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java new file mode 100644 index 0000000000..d80efc93c5 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.Const; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.resource.IResourceNaming; +import org.apache.hop.resource.ResourceDefinition; + +/** + * Metadata for the Advanced XML Output transform: writes input rows to one or more XML files + * following a hierarchical, user-defined XML tree (with one row-loop element and optional group-by + * ancestors). + */ +@Transform( + id = "AdvancedXMLOutput", + image = "AXO.svg", + name = "i18n::AdvancedXMLOutput.name", + description = "i18n::AdvancedXMLOutput.description", + categoryDescription = "i18n::AdvancedXMLOutput.category", + keywords = "i18n::AdvancedXmlOutputMeta.keyword", + documentationUrl = "/pipeline/transforms/advancedxmloutput.html", + isIncludeJdbcDrivers = false) +@Getter +@Setter +public class AdvancedXmlOutputMeta + extends BaseTransformMeta { + + private static final Class PKG = AdvancedXmlOutputMeta.class; + + /** Filename / split / zip / result options. */ + @HopMetadataProperty(key = "file") + private XmlFileOutputSupport fileSupport; + + /** Output character encoding. */ + @HopMetadataProperty( + key = "encoding", + injectionKey = "ENCODING", + injectionKeyDescription = "AdvancedXMLOutput.Injection.ENCODING") + private String encoding; + + /** When true, suppress whitespace and EOL between elements. */ + @HopMetadataProperty( + key = "compact_file", + injectionKey = "COMPACT_FILE", + injectionKeyDescription = "AdvancedXMLOutput.Injection.COMPACT_FILE") + private boolean compactFile; + + /** Add a blank line after the {@code } declaration. */ + @HopMetadataProperty( + key = "blank_line_after_xml_decl", + injectionKey = "BLANK_LINE_AFTER_XML_DECL", + injectionKeyDescription = "AdvancedXMLOutput.Injection.BLANK_LINE_AFTER_XML_DECL") + private boolean blankLineAfterXmlDeclaration; + + /** When true, an emitted attribute keeps its tag even if the source value is null. */ + @HopMetadataProperty( + key = "create_attr_if_null", + injectionKey = "CREATE_ATTR_IF_NULL", + injectionKeyDescription = "AdvancedXMLOutput.Injection.CREATE_ATTR_IF_NULL") + private boolean createAttributeIfNull; + + /** When true, an attribute with no mapped field is still emitted (with empty / default value). */ + @HopMetadataProperty( + key = "create_attr_if_unmapped", + injectionKey = "CREATE_ATTR_IF_UNMAPPED", + injectionKeyDescription = "AdvancedXMLOutput.Injection.CREATE_ATTR_IF_UNMAPPED") + private boolean createAttributeIfUnmapped; + + /** When true, an element with no mapped field is still emitted as an empty tag. */ + @HopMetadataProperty( + key = "create_empty_element", + injectionKey = "CREATE_EMPTY_ELEMENT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.CREATE_EMPTY_ELEMENT") + private boolean createEmptyElement; + + /** Trim leading/trailing whitespace on emitted text values. */ + @HopMetadataProperty( + key = "trim_values", + injectionKey = "TRIM_VALUES", + injectionKeyDescription = "AdvancedXMLOutput.Injection.TRIM_VALUES") + private boolean trimValues; + + /** Optional global decimal separator override (per-node still wins). */ + @HopMetadataProperty( + key = "default_decimal_separator", + injectionKey = "DEFAULT_DECIMAL_SEPARATOR", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DEFAULT_DECIMAL_SEPARATOR") + private String defaultDecimalSeparator; + + /** Optional global grouping separator override (per-node still wins). */ + @HopMetadataProperty( + key = "default_grouping_separator", + injectionKey = "DEFAULT_GROUPING_SEPARATOR", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DEFAULT_GROUPING_SEPARATOR") + private String defaultGroupingSeparator; + + /** When true and any node declares a namespace, write a sibling .xsd next to the output file. */ + @HopMetadataProperty( + key = "generate_xsd", + injectionKey = "GENERATE_XSD", + injectionKeyDescription = "AdvancedXMLOutput.Injection.GENERATE_XSD") + private boolean generateXsd; + + /** Optional DOCTYPE root element name. When set, emit DOCTYPE between the XML decl and root. */ + @HopMetadataProperty( + key = "doctype_root", + injectionKey = "DOCTYPE_ROOT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DOCTYPE_ROOT") + private String doctypeRootElement; + + @HopMetadataProperty( + key = "doctype_system", + injectionKey = "DOCTYPE_SYSTEM", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DOCTYPE_SYSTEM") + private String doctypeSystemId; + + @HopMetadataProperty( + key = "doctype_public", + injectionKey = "DOCTYPE_PUBLIC", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DOCTYPE_PUBLIC") + private String doctypePublicId; + + /** + * Optional XSL stylesheet href to emit as an {@code } processing instruction. + */ + @HopMetadataProperty( + key = "xsl_stylesheet_href", + injectionKey = "XSL_HREF", + injectionKeyDescription = "AdvancedXMLOutput.Injection.XSL_HREF") + private String xslStylesheetHref; + + @HopMetadataProperty( + key = "xsl_stylesheet_type", + injectionKey = "XSL_TYPE", + injectionKeyDescription = "AdvancedXMLOutput.Injection.XSL_TYPE") + private String xslStylesheetType; + + /** The hierarchical XML tree definition. */ + @HopMetadataProperty(key = "tree") + private XmlNode rootNode; + + public AdvancedXmlOutputMeta() { + super(); + this.fileSupport = new XmlFileOutputSupport(); + this.encoding = Const.UTF_8; + this.blankLineAfterXmlDeclaration = true; + this.createEmptyElement = true; + this.xslStylesheetType = "text/xsl"; + this.rootNode = defaultRootNode(); + } + + public AdvancedXmlOutputMeta(AdvancedXmlOutputMeta m) { + this(); + this.fileSupport = new XmlFileOutputSupport(m.fileSupport); + this.encoding = m.encoding; + this.compactFile = m.compactFile; + this.blankLineAfterXmlDeclaration = m.blankLineAfterXmlDeclaration; + this.createAttributeIfNull = m.createAttributeIfNull; + this.createAttributeIfUnmapped = m.createAttributeIfUnmapped; + this.createEmptyElement = m.createEmptyElement; + this.trimValues = m.trimValues; + this.defaultDecimalSeparator = m.defaultDecimalSeparator; + this.defaultGroupingSeparator = m.defaultGroupingSeparator; + this.generateXsd = m.generateXsd; + this.doctypeRootElement = m.doctypeRootElement; + this.doctypeSystemId = m.doctypeSystemId; + this.doctypePublicId = m.doctypePublicId; + this.xslStylesheetHref = m.xslStylesheetHref; + this.xslStylesheetType = m.xslStylesheetType; + this.rootNode = m.rootNode == null ? defaultRootNode() : new XmlNode(m.rootNode); + } + + @Override + public Object clone() { + return new AdvancedXmlOutputMeta(this); + } + + /** Default tree: {@code } so a fresh transform validates. */ + private static XmlNode defaultRootNode() { + XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); + XmlNode loop = new XmlNode("Row", XmlNode.NodeKind.Element); + loop.setLoop(true); + root.addChild(loop); + return root; + } + + @Override + public void getFields( + IRowMeta row, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) { + // No fields are added to the output stream; rows pass through unchanged. + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + + CheckResult cr; + + // Tree: at least one node, exactly one loop node, mapped fields exist + List validationErrors = AdvancedXmlOutputValidator.validate(rootNode, prev); + if (validationErrors.isEmpty()) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "AdvancedXMLOutputMeta.CheckResult.TreeOk"), + transformMeta); + remarks.add(cr); + } else { + for (String err : validationErrors) { + remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, err, transformMeta)); + } + } + + if (input.length > 0) { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "AdvancedXMLOutputMeta.CheckResult.ExpectedInputOk"), + transformMeta); + remarks.add(cr); + } else { + cr = + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "AdvancedXMLOutputMeta.CheckResult.ExpectedInputError"), + transformMeta); + remarks.add(cr); + } + } + + @Override + public String exportResources( + IVariables variables, + Map definitions, + IResourceNaming resourceNamingInterface, + IHopMetadataProvider metadataProvider) + throws HopException { + try { + if (fileSupport != null && !Utils.isEmpty(fileSupport.getFileName())) { + FileObject fileObject = + HopVfs.getFileObject(variables.resolve(fileSupport.getFileName()), variables); + fileSupport.setFileName(resourceNamingInterface.nameResource(fileObject, variables, true)); + } + return null; + } catch (Exception e) { + throw new HopException(e); + } + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputValidator.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputValidator.java new file mode 100644 index 0000000000..9a8406c6fc --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputValidator.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.util.Utils; + +/** + * Static structural and semantic validation of an {@link XmlNode} tree against the upstream row + * metadata. Returns a list of human-readable error messages (empty when the tree is valid). + */ +final class AdvancedXmlOutputValidator { + + private AdvancedXmlOutputValidator() {} + + static List validate(XmlNode root, IRowMeta inputRowMeta) { + List errors = new ArrayList<>(); + if (root == null) { + errors.add("XML tree is empty - configure the XML tree first."); + return errors; + } + if (root.getKind() != XmlNode.NodeKind.Element) { + errors.add("Root node must be an element."); + } + if (Utils.isEmpty(root.getName())) { + errors.add("Root element must have a name."); + } + + int[] loopCount = new int[1]; + walk(root, null, loopCount, inputRowMeta, errors); + + if (loopCount[0] == 0) { + errors.add( + "No loop element configured. Mark exactly one element in the XML tree as the loop."); + } else if (loopCount[0] > 1) { + errors.add( + "Multiple loop elements configured (" + + loopCount[0] + + "). Exactly one element in the XML tree may be the loop."); + } + + return errors; + } + + private static void walk( + XmlNode node, XmlNode parent, int[] loopCount, IRowMeta inputRowMeta, List errors) { + + if (Utils.isEmpty(node.getName())) { + errors.add( + "Node has no name (parent: " + (parent == null ? "" : parent.getName()) + ")"); + } + + if (node.isLoop()) { + loopCount[0]++; + if (node.getKind() != XmlNode.NodeKind.Element) { + errors.add("Loop node '" + node.getName() + "' must be an element."); + } + } + + if (node.isGroupBy() && (node.getMappedField() == null || node.getMappedField().isEmpty())) { + errors.add( + "Group-by node '" + node.getName() + "' must reference an input field as its key."); + } + + if (!Utils.isEmpty(node.getMappedField()) + && inputRowMeta != null + && inputRowMeta.indexOfValue(node.getMappedField()) < 0) { + errors.add( + "Node '" + + node.getName() + + "' references unknown input field '" + + node.getMappedField() + + "'."); + } + + if (node.getChildren() != null) { + // Attributes must come from an Element parent + if (node.getKind() != XmlNode.NodeKind.Element && !node.getChildren().isEmpty()) { + errors.add( + "Non-element node '" + + node.getName() + + "' cannot have children (kind=" + + node.getKind() + + ")."); + } + for (XmlNode child : node.getChildren()) { + walk(child, node, loopCount, inputRowMeta, errors); + } + } + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java new file mode 100644 index 0000000000..ebd56c3138 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** + * Self-contained holder of all filename / extension / split / date-time / zip / result-file options + * used by {@link AdvancedXmlOutputMeta}. Kept as a nested-style POJO so it serializes under a + * single {@code } element, mirroring the convention used by the existing XML Output + * transform. + */ +@Getter +@Setter +public class XmlFileOutputSupport { + + /** Base name of the output file. */ + @HopMetadataProperty( + key = "name", + injectionKey = "FILENAME", + injectionKeyDescription = "AdvancedXMLOutput.Injection.FILENAME") + private String fileName; + + /** Optional file extension (without leading dot). */ + @HopMetadataProperty( + key = "extension", + injectionKey = "EXTENSION", + injectionKeyDescription = "AdvancedXMLOutput.Injection.EXTENSION") + private String extension; + + /** Maximum number of input rows per file. 0 = unlimited (single file). */ + @HopMetadataProperty( + key = "splitevery", + injectionKey = "SPLIT_EVERY", + injectionKeyDescription = "AdvancedXMLOutput.Injection.SPLIT_EVERY") + private int splitEvery; + + /** Add the transform copy number to the filename. */ + @HopMetadataProperty( + key = "split", + injectionKey = "INC_TRANSFORMNR_IN_FILENAME", + injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_TRANSFORMNR_IN_FILENAME") + private boolean transformNrInFilename; + + /** Add the date (yyyyMMdd) to the filename. */ + @HopMetadataProperty( + key = "add_date", + injectionKey = "INC_DATE_IN_FILENAME", + injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_DATE_IN_FILENAME") + private boolean dateInFilename; + + /** Add the time (HHmmss) to the filename. */ + @HopMetadataProperty( + key = "add_time", + injectionKey = "INC_TIME_IN_FILENAME", + injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_TIME_IN_FILENAME") + private boolean timeInFilename; + + /** Wrap the destination file in a zip archive. */ + @HopMetadataProperty( + key = "zipped", + injectionKey = "ZIPPED", + injectionKeyDescription = "AdvancedXMLOutput.Injection.ZIPPED") + private boolean zipped; + + /** Add the produced filename(s) to the pipeline result file list. */ + @HopMetadataProperty( + key = "add_to_result_filenames", + injectionKey = "ADD_TO_RESULT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.ADD_TO_RESULT") + private boolean addToResultFilenames; + + /** Defer file creation until the first input row is received. */ + @HopMetadataProperty( + key = "do_not_open_newfile_init", + injectionKey = "DO_NOT_CREATE_FILE_AT_STARTUP", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DO_NOT_CREATE_FILE_AT_STARTUP") + private boolean doNotOpenNewFileInit; + + /** Delete the output file at the end of the run if no rows were written. */ + @HopMetadataProperty( + key = "do_not_create_empty_file", + injectionKey = "DO_NOT_CREATE_EMPTY_FILE", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DO_NOT_CREATE_EMPTY_FILE") + private boolean doNotCreateEmptyFile; + + /** Use a custom date-time pattern instead of the date/time flags above. */ + @HopMetadataProperty( + key = "SpecifyFormat", + injectionKey = "SPEFICY_FORMAT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.SPEFICY_FORMAT") + private boolean specifyFormat; + + @HopMetadataProperty( + key = "date_time_format", + injectionKey = "DATE_FORMAT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.DATE_FORMAT") + private String dateTimeFormat; + + public XmlFileOutputSupport() { + fileName = ""; + extension = "xml"; + splitEvery = 0; + } + + public XmlFileOutputSupport(XmlFileOutputSupport o) { + this(); + this.fileName = o.fileName; + this.extension = o.extension; + this.splitEvery = o.splitEvery; + this.transformNrInFilename = o.transformNrInFilename; + this.dateInFilename = o.dateInFilename; + this.timeInFilename = o.timeInFilename; + this.zipped = o.zipped; + this.addToResultFilenames = o.addToResultFilenames; + this.doNotOpenNewFileInit = o.doNotOpenNewFileInit; + this.doNotCreateEmptyFile = o.doNotCreateEmptyFile; + this.specifyFormat = o.specifyFormat; + this.dateTimeFormat = o.dateTimeFormat; + } + + /** + * Build the resolved filename for a particular copy/split combination. + * + * @param variables variable resolver + * @param copyNr the transform copy number (only used when {@link #transformNrInFilename} is set) + * @param splitNr the split sequence number (only used when {@link #splitEvery} > 0) + * @param zipArchive when true and {@link #zipped} is set, returns the .zip archive name; when + * false, returns the inner file name with the configured extension. + */ + public String buildFilename(IVariables variables, int copyNr, int splitNr, boolean zipArchive) { + SimpleDateFormat daf = new SimpleDateFormat(); + DecimalFormat df = new DecimalFormat("00000"); + + String filename = variables.resolve(fileName); + String realExtension = variables.resolve(extension); + Date now = new Date(); + + if (specifyFormat && !Utils.isEmpty(dateTimeFormat)) { + daf.applyPattern(dateTimeFormat); + filename += daf.format(now); + } else { + if (dateInFilename) { + daf.applyPattern("yyyyMMdd"); + filename += "_" + daf.format(now); + } + if (timeInFilename) { + daf.applyPattern("HHmmss"); + filename += "_" + daf.format(now); + } + } + + if (transformNrInFilename) { + filename += "_" + copyNr; + } + if (splitEvery > 0) { + filename += "_" + df.format(splitNr + 1); + } + + if (zipped) { + if (zipArchive) { + filename += ".zip"; + } else if (!Utils.isEmpty(realExtension)) { + filename += "." + realExtension; + } + } else if (!Utils.isEmpty(realExtension)) { + filename += "." + realExtension; + } + return filename; + } + + /** Returns up to {@code nr} sample filenames for use in the dialog's "Show files" preview. */ + public String[] previewFilenames(IVariables variables) { + int copies = transformNrInFilename ? 3 : 1; + int splits = splitEvery != 0 ? 3 : 1; + int nr = copies * splits; + if (nr > 1) { + nr++; + } + String[] fileNames = new String[nr]; + int i = 0; + for (int copy = 0; copy < copies; copy++) { + for (int split = 0; split < splits; split++) { + fileNames[i++] = buildFilename(variables, copy, split, false); + } + } + if (i < nr) { + fileNames[i] = "..."; + } + return fileNames; + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java new file mode 100644 index 0000000000..595f711d6e --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import com.google.common.base.Enums; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** + * Describes a single node in the hierarchical XML tree of the Advanced XML Output transform. + * + *

An {@code XmlNode} can represent an element or an attribute. Elements may have any number of + * child nodes (recursive structure). Exactly one element in the tree must be marked as the loop + * node ({@link #loop}) to indicate where input rows are emitted. Optionally, ancestors of the loop + * node can be marked as group-by ({@link #groupBy}) to collapse consecutive rows that share the + * same group key into a single occurrence of the group element. + */ +@Getter +@Setter +public class XmlNode { + + @SuppressWarnings("java:S115") + public enum NodeKind { + /** Standard XML element with optional text content and/or children. */ + Element, + /** XML attribute on the parent element. */ + Attribute, + /** Pre-built XML fragment inserted as parsed nodes (requires the source field to hold XML). */ + DocumentFragment; + + public static NodeKind getIfPresent(String name) { + return Enums.getIfPresent(NodeKind.class, name).or(Element); + } + } + + /** Local name of the element or attribute. */ + @HopMetadataProperty(key = "name") + private String name; + + /** Optional XML namespace URI for this element (ignored for attributes in v1). */ + @HopMetadataProperty(key = "namespace") + private String namespace; + + /** Element / Attribute / DocumentFragment. */ + @HopMetadataProperty(key = "kind") + private NodeKind kind; + + /** + * Name of the input field whose value provides this node's content (for an element) or value (for + * an attribute / document fragment). Empty / null means "static node". + */ + @HopMetadataProperty(key = "mapped_field") + private String mappedField; + + /** Static text used when {@link #mappedField} is empty (or the field value is null). */ + @HopMetadataProperty(key = "default_value") + private String defaultValue; + + /** Optional Hop value-type override (one of {@link ValueMetaBase} type codes). 0 = auto. */ + @HopMetadataProperty(key = "type", intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class) + private int type; + + /** Optional conversion mask. */ + @HopMetadataProperty(key = "format") + private String format; + + @HopMetadataProperty(key = "length") + private int length; + + @HopMetadataProperty(key = "precision") + private int precision; + + @HopMetadataProperty(key = "currency") + private String currencySymbol; + + @HopMetadataProperty(key = "decimal") + private String decimalSymbol; + + @HopMetadataProperty(key = "group") + private String groupingSymbol; + + /** Output this node even when its value is null. */ + @HopMetadataProperty(key = "force_create") + private boolean forceCreate; + + /** Marks this element as the row-loop element (exactly one in the tree). */ + @HopMetadataProperty(key = "loop") + private boolean loop; + + /** + * Marks this element as a group-by ancestor of the loop element. Consecutive input rows that + * share the same value of {@link #mappedField} are emitted under a single occurrence of this + * element. + */ + @HopMetadataProperty(key = "group_by") + private boolean groupBy; + + /** Children (only meaningful for {@link NodeKind#Element}). */ + @HopMetadataProperty(key = "node", groupKey = "children", isExcludedFromInjection = true) + private List children; + + public XmlNode() { + this.kind = NodeKind.Element; + this.length = -1; + this.precision = -1; + this.type = 0; + this.children = new ArrayList<>(); + } + + public XmlNode(String name, NodeKind kind) { + this(); + this.name = name; + this.kind = kind; + } + + public XmlNode(XmlNode other) { + this(); + this.name = other.name; + this.namespace = other.namespace; + this.kind = other.kind; + this.mappedField = other.mappedField; + this.defaultValue = other.defaultValue; + this.type = other.type; + this.format = other.format; + this.length = other.length; + this.precision = other.precision; + this.currencySymbol = other.currencySymbol; + this.decimalSymbol = other.decimalSymbol; + this.groupingSymbol = other.groupingSymbol; + this.forceCreate = other.forceCreate; + this.loop = other.loop; + this.groupBy = other.groupBy; + if (other.children != null) { + for (XmlNode c : other.children) { + this.children.add(new XmlNode(c)); + } + } + } + + /** Convenience: returns true if this node has any direct children of element kind. */ + public boolean hasElementChildren() { + if (children == null) { + return false; + } + for (XmlNode c : children) { + if (c.kind == NodeKind.Element || c.kind == NodeKind.DocumentFragment) { + return true; + } + } + return false; + } + + /** Convenience: returns true if this node has any direct children of attribute kind. */ + public boolean hasAttributeChildren() { + if (children == null) { + return false; + } + for (XmlNode c : children) { + if (c.kind == NodeKind.Attribute) { + return true; + } + } + return false; + } + + public void addChild(XmlNode child) { + if (children == null) { + children = new ArrayList<>(); + } + children.add(child); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof XmlNode that)) { + return false; + } + return loop == that.loop + && groupBy == that.groupBy + && forceCreate == that.forceCreate + && type == that.type + && length == that.length + && precision == that.precision + && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) + && kind == that.kind + && Objects.equals(mappedField, that.mappedField) + && Objects.equals(defaultValue, that.defaultValue) + && Objects.equals(format, that.format) + && Objects.equals(currencySymbol, that.currencySymbol) + && Objects.equals(decimalSymbol, that.decimalSymbol) + && Objects.equals(groupingSymbol, that.groupingSymbol) + && Objects.equals(children, that.children); + } + + @Override + public int hashCode() { + return Objects.hash(name, kind, loop, groupBy, mappedField); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(kind == NodeKind.Attribute ? "@" : "<").append(name == null ? "" : name); + if (loop) { + sb.append("[loop]"); + } + if (groupBy) { + sb.append("[group]"); + } + if (mappedField != null && !mappedField.isEmpty()) { + sb.append("=").append(mappedField); + } + return sb.toString(); + } +} diff --git a/plugins/transforms/xml/src/main/resources/AXO.svg b/plugins/transforms/xml/src/main/resources/AXO.svg new file mode 100644 index 0000000000..6509591a51 --- /dev/null +++ b/plugins/transforms/xml/src/main/resources/AXO.svg @@ -0,0 +1,38 @@ + + + + + + + + + AXO + + + + + + + + + + diff --git a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties new file mode 100644 index 0000000000..3f7f5ab6c8 --- /dev/null +++ b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +AdvancedXMLOutput.name=Advanced XML output +AdvancedXMLOutput.description=Write rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. +AdvancedXMLOutput.category=Output +AdvancedXmlOutputMeta.keyword=xml,output,write,tree,hierarchical,advanced + +AdvancedXMLOutput.Injection.FILENAME=The base name of the XML file. +AdvancedXMLOutput.Injection.EXTENSION=Extension appended to the file name (without leading dot). +AdvancedXMLOutput.Injection.SPLIT_EVERY=Maximum rows per file before splitting (0 = no split). +AdvancedXMLOutput.Injection.INC_TRANSFORMNR_IN_FILENAME=Add the transform copy number to the filename. +AdvancedXMLOutput.Injection.INC_DATE_IN_FILENAME=Add the system date (yyyyMMdd) to the filename. +AdvancedXMLOutput.Injection.INC_TIME_IN_FILENAME=Add the system time (HHmmss) to the filename. +AdvancedXMLOutput.Injection.ZIPPED=Wrap the output file in a zip archive. +AdvancedXMLOutput.Injection.ADD_TO_RESULT=Add produced filename(s) to the pipeline result file list. +AdvancedXMLOutput.Injection.DO_NOT_CREATE_FILE_AT_STARTUP=Defer file creation until the first row is received. +AdvancedXMLOutput.Injection.DO_NOT_CREATE_EMPTY_FILE=Delete the file at the end of the run if no rows were written. +AdvancedXMLOutput.Injection.SPEFICY_FORMAT=Use the custom date/time pattern below instead of the date/time flags. +AdvancedXMLOutput.Injection.DATE_FORMAT=Custom date/time pattern for the filename. +AdvancedXMLOutput.Injection.ENCODING=Output character encoding (defaults to UTF-8). +AdvancedXMLOutput.Injection.COMPACT_FILE=Suppress whitespace and EOL between elements. +AdvancedXMLOutput.Injection.BLANK_LINE_AFTER_XML_DECL=Add a blank line right after the XML declaration. +AdvancedXMLOutput.Injection.CREATE_ATTR_IF_NULL=Emit attributes whose source value is null. +AdvancedXMLOutput.Injection.CREATE_ATTR_IF_UNMAPPED=Emit attributes that have no mapped field (using their default value). +AdvancedXMLOutput.Injection.CREATE_EMPTY_ELEMENT=Emit empty start/end tags for elements with no value. +AdvancedXMLOutput.Injection.TRIM_VALUES=Trim leading/trailing whitespace from emitted values. +AdvancedXMLOutput.Injection.DEFAULT_DECIMAL_SEPARATOR=Default decimal separator for numeric values (per-node still wins). +AdvancedXMLOutput.Injection.DEFAULT_GROUPING_SEPARATOR=Default grouping separator for numeric values (per-node still wins). +AdvancedXMLOutput.Injection.GENERATE_XSD=Generate a sibling .xsd file alongside the output XML. +AdvancedXMLOutput.Injection.DOCTYPE_ROOT=Root element name for the DOCTYPE declaration. +AdvancedXMLOutput.Injection.DOCTYPE_SYSTEM=System identifier for the DOCTYPE declaration. +AdvancedXMLOutput.Injection.DOCTYPE_PUBLIC=Public identifier for the DOCTYPE declaration. +AdvancedXMLOutput.Injection.XSL_HREF=URL of an XSL stylesheet to reference via xml-stylesheet PI. +AdvancedXMLOutput.Injection.XSL_TYPE=MIME type for the XSL stylesheet PI (defaults to text/xsl). + +AdvancedXMLOutputMeta.CheckResult.TreeOk=The XML tree is structurally valid. +AdvancedXMLOutputMeta.CheckResult.ExpectedInputOk=Transform is connected to a previous transform receiving rows. +AdvancedXMLOutputMeta.CheckResult.ExpectedInputError=No input received from previous transforms. + +AdvancedXMLOutputDialog.DialogTitle=Advanced XML output +AdvancedXMLOutputDialog.Filename.Label=Filename +AdvancedXMLOutputDialog.Extension.Label=Extension +AdvancedXMLOutputDialog.Encoding.Label=Encoding +AdvancedXMLOutputDialog.AddFileToResult.Label=Add filename to result +AdvancedXMLOutputDialog.Notice.Label=Note: the XML tree designer will be added in a follow-up release. For now, configure the tree by editing the transform metadata directly. diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java new file mode 100644 index 0000000000..ff5b3fb8cf --- /dev/null +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AdvancedXmlOutputMetaTest { + + @BeforeEach + void beforeEach() throws Exception { + HopClientEnvironment.init(); + } + + @Test + void testSerializationRoundTrip() throws Exception { + AdvancedXmlOutputMeta meta = + TransformSerializationTestUtil.testSerialization( + "/advanced-xml-output.xml", AdvancedXmlOutputMeta.class); + + assertEquals("UTF-8", meta.getEncoding()); + assertEquals("/tmp/orders", meta.getFileSupport().getFileName()); + assertEquals("xml", meta.getFileSupport().getExtension()); + assertTrue(meta.getFileSupport().isAddToResultFilenames()); + assertTrue(meta.getFileSupport().isDoNotCreateEmptyFile()); + assertTrue(meta.isCreateEmptyElement()); + assertTrue(meta.isBlankLineAfterXmlDeclaration()); + + XmlNode root = meta.getRootNode(); + assertNotNull(root); + assertEquals("orders", root.getName()); + assertEquals(1, root.getChildren().size()); + + XmlNode order = root.getChildren().get(0); + assertEquals("order", order.getName()); + assertTrue(order.isGroupBy()); + assertEquals("orderId", order.getMappedField()); + assertEquals(2, order.getChildren().size()); + + XmlNode id = order.getChildren().get(0); + assertEquals("id", id.getName()); + assertEquals(XmlNode.NodeKind.Attribute, id.getKind()); + assertEquals("orderId", id.getMappedField()); + + XmlNode item = order.getChildren().get(1); + assertEquals("item", item.getName()); + assertTrue(item.isLoop()); + assertEquals(2, item.getChildren().size()); + + XmlNode price = item.getChildren().get(1); + assertEquals("price", price.getName()); + assertEquals("price", price.getMappedField()); + assertEquals("0.00", price.getFormat()); + } + + @Test + void testCloneCreatesIndependentTree() { + AdvancedXmlOutputMeta original = new AdvancedXmlOutputMeta(); + original.getRootNode().setName("Original"); + original.getRootNode().getChildren().get(0).setName("OriginalLoop"); + + AdvancedXmlOutputMeta copy = (AdvancedXmlOutputMeta) original.clone(); + + assertEquals("Original", copy.getRootNode().getName()); + assertEquals("OriginalLoop", copy.getRootNode().getChildren().get(0).getName()); + + // Mutate original; copy must remain untouched. + original.getRootNode().setName("Mutated"); + original.getRootNode().getChildren().get(0).setName("MutatedLoop"); + + assertEquals("Original", copy.getRootNode().getName()); + assertEquals("OriginalLoop", copy.getRootNode().getChildren().get(0).getName()); + } + + @Test + void testDefaultMetaValidates() { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + java.util.List errors = AdvancedXmlOutputValidator.validate(meta.getRootNode(), null); + assertEquals(0, errors.size(), "Default tree should validate cleanly: " + errors); + } + + @Test + void testValidatorRejectsTreeWithoutLoop() { + XmlNode root = new XmlNode("root", XmlNode.NodeKind.Element); + root.addChild(new XmlNode("child", XmlNode.NodeKind.Element)); + java.util.List errors = AdvancedXmlOutputValidator.validate(root, null); + assertTrue( + errors.stream().anyMatch(e -> e.toLowerCase().contains("loop")), + "Expected validation error about missing loop, got: " + errors); + } + + @Test + void testValidatorRejectsTreeWithMultipleLoops() { + XmlNode root = new XmlNode("root", XmlNode.NodeKind.Element); + XmlNode a = new XmlNode("a", XmlNode.NodeKind.Element); + a.setLoop(true); + XmlNode b = new XmlNode("b", XmlNode.NodeKind.Element); + b.setLoop(true); + root.addChild(a); + root.addChild(b); + java.util.List errors = AdvancedXmlOutputValidator.validate(root, null); + assertTrue( + errors.stream().anyMatch(e -> e.toLowerCase().contains("multiple loop")), + "Expected validation error about multiple loops, got: " + errors); + } +} diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java new file mode 100644 index 0000000000..d90903845a --- /dev/null +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.RowMetaAndData; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transforms.xml.PipelineTestFactory; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end runtime tests for the Advanced XML Output transform. Each test runs an actual pipeline + * via the local engine, writes to a temp file and asserts on the produced XML. + */ +class AdvancedXmlOutputTest { + + @BeforeAll + static void setup() throws Exception { + HopEnvironment.init(); + } + + // --------------------------------------------------------------------------- + // Simple flat case (no group-by) + // --------------------------------------------------------------------------- + + @Test + void testFlatTreeProducesOneRowPerInputLine(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("flat"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + + runPipeline(meta, buildFlatRows("Alice", 30, "Bob", 25)); + + String xml = readWrittenFile(output); + // Expect Alice30... + assertTrue(xml.contains("")); + assertTrue(xml.contains("")); + assertEquals(2, count(xml, "")); + assertTrue(xml.contains("Alice")); + assertTrue(xml.contains("Bob")); + assertTrue(xml.contains("30")); + assertTrue(xml.contains("25")); + } + + // --------------------------------------------------------------------------- + // Group-by case + // --------------------------------------------------------------------------- + + @Test + void testGroupByCollapsesConsecutiveRowsWithSameKey(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("orders"); + AdvancedXmlOutputMeta meta = buildOrderMeta(output.toString()); + + // 3 rows: order=1 (foo, bar), order=2 (baz) + List rows = new ArrayList<>(); + rows.add(orderRow(1L, "foo", 1.50)); + rows.add(orderRow(1L, "bar", 2.00)); + rows.add(orderRow(2L, "baz", 3.25)); + + runPipeline(meta, rows); + + String xml = readWrittenFile(output); + // Two elements (one per group), three elements (one per row). + assertEquals(2, count(xml, "")); + assertEquals(3, count(xml, "")); + // Group attribute on order element + assertTrue(xml.contains("id=\"1\"")); + assertTrue(xml.contains("id=\"2\"")); + // Items appear under the right group (order=1 contains foo and bar; order=2 contains baz) + int firstOrderEnd = xml.indexOf(""); + int secondOrderStart = xml.indexOf("foo")); + assertTrue(firstOrderXml.contains("bar")); + assertFalse(firstOrderXml.contains("baz")); + assertTrue(secondOrderXml.contains("baz")); + } + + // --------------------------------------------------------------------------- + // Don't-create-empty-file + // --------------------------------------------------------------------------- + + @Test + void testDontCreateEmptyFileSkipsFileWhenNoRows(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("empty"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.getFileSupport().setDoNotCreateEmptyFile(true); + + runPipeline(meta, new ArrayList<>()); + + Path expected = Path.of(output.toString() + ".xml"); + assertFalse(Files.exists(expected), "Expected no file to be created when input is empty"); + } + + // --------------------------------------------------------------------------- + // Compact mode + // --------------------------------------------------------------------------- + + @Test + void testCompactFileHasNoNewlinesBetweenElements(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("compact"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.setCompactFile(true); + meta.setBlankLineAfterXmlDeclaration(false); + + runPipeline(meta, buildFlatRows("X", 1, "Y", 2)); + + String xml = readWrittenFile(output); + // After the XML declaration, no newlines between row elements should remain. + int afterDecl = xml.indexOf("?>") + 2; + String body = xml.substring(afterDecl); + assertFalse(body.contains("\n"), "Compact mode should not contain newlines: " + body); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private AdvancedXmlOutputMeta buildFlatMeta(String filenameWithoutExt) { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.getFileSupport().setFileName(filenameWithoutExt); + meta.getFileSupport().setExtension("xml"); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + + XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); + XmlNode row = new XmlNode("Row", XmlNode.NodeKind.Element); + row.setLoop(true); + XmlNode name = new XmlNode("name", XmlNode.NodeKind.Element); + name.setMappedField("name"); + XmlNode age = new XmlNode("age", XmlNode.NodeKind.Element); + age.setMappedField("age"); + row.addChild(name); + row.addChild(age); + root.addChild(row); + meta.setRootNode(root); + return meta; + } + + private List buildFlatRows(Object... pairs) { + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("name")); + rm.addValueMeta(new ValueMetaInteger("age")); + List rows = new ArrayList<>(); + for (int i = 0; i < pairs.length; i += 2) { + rows.add(new RowMetaAndData(rm, pairs[i], Long.valueOf(((Number) pairs[i + 1]).longValue()))); + } + return rows; + } + + private AdvancedXmlOutputMeta buildOrderMeta(String filenameWithoutExt) { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.getFileSupport().setFileName(filenameWithoutExt); + meta.getFileSupport().setExtension("xml"); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + + // + // + // + // {itemName} + // {price} + // + // + // + XmlNode orders = new XmlNode("orders", XmlNode.NodeKind.Element); + + XmlNode order = new XmlNode("order", XmlNode.NodeKind.Element); + order.setGroupBy(true); + order.setMappedField("orderId"); + + XmlNode id = new XmlNode("id", XmlNode.NodeKind.Attribute); + id.setMappedField("orderId"); + order.addChild(id); + + XmlNode item = new XmlNode("item", XmlNode.NodeKind.Element); + item.setLoop(true); + XmlNode itemName = new XmlNode("name", XmlNode.NodeKind.Element); + itemName.setMappedField("itemName"); + XmlNode price = new XmlNode("price", XmlNode.NodeKind.Element); + price.setMappedField("price"); + price.setFormat("0.00"); + item.addChild(itemName); + item.addChild(price); + order.addChild(item); + + orders.addChild(order); + meta.setRootNode(orders); + return meta; + } + + private RowMetaAndData orderRow(long orderId, String itemName, double price) { + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaInteger("orderId")); + rm.addValueMeta(new ValueMetaString("itemName")); + rm.addValueMeta(new ValueMetaNumber("price")); + return new RowMetaAndData(rm, orderId, itemName, price); + } + + private void runPipeline(AdvancedXmlOutputMeta meta, List input) + throws Exception { + PipelineMeta pipelineMeta = PipelineTestFactory.generateTestTransformation(null, meta, "axo"); + PipelineTestFactory.executeTestTransformation( + pipelineMeta, + PipelineTestFactory.INJECTOR_TRANSFORMNAME, + "axo", + PipelineTestFactory.DUMMY_TRANSFORMNAME, + input); + } + + private String readWrittenFile(Path withoutExt) throws Exception { + Path withExt = Path.of(withoutExt.toString() + ".xml"); + return Files.readString(withExt); + } + + private int count(String haystack, String needle) { + int c = 0; + int idx = 0; + while ((idx = haystack.indexOf(needle, idx)) >= 0) { + c++; + idx += needle.length(); + } + return c; + } +} diff --git a/plugins/transforms/xml/src/test/resources/advanced-xml-output.xml b/plugins/transforms/xml/src/test/resources/advanced-xml-output.xml new file mode 100644 index 0000000000..03f06d7e46 --- /dev/null +++ b/plugins/transforms/xml/src/test/resources/advanced-xml-output.xml @@ -0,0 +1,83 @@ + + + + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + /tmp/orders + xml + 0 + N + N + N + N + Y + N + Y + N + + + + orders + Element + + + order + Element + Y + orderId + + + id + Attribute + orderId + + + item + Element + Y + + + name + Element + itemName + + + price + Element + price + 0.00 + + + + + + + + From 042b04a0042ef4147269a72a8e58a457b9c87fa1 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Fri, 8 May 2026 14:40:57 +0200 Subject: [PATCH 2/9] first working version of the Advanced XML Output transform --- .../transforms/advancedxmloutput.adoc | 145 ++- .../advancedxmloutput/AdvancedXmlOutput.java | 50 +- .../AdvancedXmlOutputDialog.java | 841 ++++++++++++++-- .../AdvancedXmlOutputMeta.java | 2 +- .../AdvancedXmlOutputXsdWriter.java | 190 ++++ .../XmlFileOutputSupport.java | 18 + .../advancedxmloutput/XmlTreeDesigner.java | 939 ++++++++++++++++++ .../messages/messages_en_US.properties | 84 +- .../transforms/advanced-xml-output-basic.hpl | 260 +++++ .../advanced-xml-output-grouped.hpl | 359 +++++++ .../AdvancedXmlOutputSamplesTest.java | 164 +++ .../AdvancedXmlOutputTest.java | 149 +++ .../AdvancedXmlOutputXsdWriterTest.java | 174 ++++ 13 files changed, 3219 insertions(+), 156 deletions(-) create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriter.java create mode 100644 plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java create mode 100644 plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl create mode 100644 plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl create mode 100644 plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java create mode 100644 plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriterTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc index d511979ae0..5a0e4669be 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc @@ -29,7 +29,7 @@ The Advanced XML Output transform writes rows from any source to one or more XML The XML tree is a recursive structure of elements, attributes and document-fragment nodes. Exactly one element in the tree must be marked as the row-*loop*: each input row produces one occurrence of that element with its full subtree. Optionally, ancestors of the loop can be marked as *group-by*: consecutive input rows that share the same group key are emitted under a single occurrence of the group element. -This transform complements the simpler `XML Output` transform. `XML Output` is the right choice when you want a flat document of repeating rows; `Advanced XML Output` is the right choice when you need a deeper, custom-shaped XML structure (loops nested inside groups, attributes at any level, document fragments, namespaces). +This transform complements the simpler `XML Output` transform. Use `XML Output` for a flat document of repeating rows; use `Advanced XML Output` when you need a deeper, custom-shaped XML structure (loops nested inside groups, attributes at any level, document fragments, namespaces, schema generation). | == Supported Engines @@ -44,70 +44,135 @@ This transform complements the simpler `XML Output` transform. `XML Output` is t == Options -[NOTE] -==== -The Phase-1 dialog only exposes the basic file options. The interactive XML tree designer is on the way; until it ships, configure the tree by editing the transform metadata directly (HopGui's "Show metadata" command, or by hand in your `.hpl` file). -==== +The dialog is organized into three tabs: *File*, *Content* and *XML Tree*. -=== File Tab +=== File tab [options="header"] |=== |Option|Description |Transform name|Name of the transform. -|Filename|Base name of the output XML file (without extension). +|Filename|Base name of the output XML file (without extension). VFS URIs are supported. |Extension|File extension (without the leading dot). Defaults to `xml`. -|Encoding|Character encoding for the output file. Defaults to UTF-8. -|Add filename to result|When enabled, the produced filename is added to the pipeline's result file list (only after at least one row is written). +|Encoding|Character encoding for the output file. Defaults to `UTF-8`. +|Include transform copy number in filename|Append the transform copy number to the filename. +|Include date in filename|Append the system date (`yyyyMMdd`) to the filename. +|Include time in filename|Append the system time (`HHmmss`) to the filename. +|Specify custom date/time format|Use a custom date/time pattern instead of the date/time toggles above. +|Date/time format|Java `SimpleDateFormat` pattern, used when the custom format toggle is on. +|Split every N rows|Maximum rows per file before rolling over to a new split. `0` = no splitting. +|Zip output file|Wrap each output file in a zip archive (one entry per file). Generated XSDs are written next to the archive, not inside it. +|Do not open new file at start|Defer file creation until the first input row is received. +|Do not create file if no rows|Delete the output file at the end of the run if no rows were ever written. +|Add filename to result|Add the produced file(s) to the pipeline's result file list (only after at least one row is written). +|Show file name(s) ...|Pops up a list with sample filenames built from the current settings. |=== -=== Content options - -The following advanced options are available on the metadata (and will be exposed in the dialog in the next phase): +=== Content tab [options="header"] |=== |Option|Description -|`compactFile`|Suppress whitespace and EOL between elements; useful for byte-size-sensitive output. -|`blankLineAfterXmlDeclaration`|Add a blank line right after the `` declaration (default `true`, matching common formatters). -|`createEmptyElement`|Emit an open/close tag pair for an element that has no value and no children (default `true`). -|`createAttributeIfNull`|Emit an attribute even when its source value is `null`. -|`createAttributeIfUnmapped`|Emit an attribute that has no mapped field (using its default value). -|`trimValues`|Trim leading/trailing whitespace from emitted text values. -|`defaultDecimalSeparator`|Default decimal separator for numeric values; per-node settings still take precedence. -|`defaultGroupingSeparator`|Default grouping separator for numeric values; per-node settings still take precedence. -|`generateXsd`|When set and the tree declares at least one namespace, write a sibling `.xsd` next to the output file. -|`doctypeRootElement` / `doctypeSystemId` / `doctypePublicId`|Emit a `` declaration between the XML declaration and the root element. -|`xslStylesheetHref` / `xslStylesheetType`|Emit an `` processing instruction; defaults to `text/xsl` when type is empty. -|`splitEvery`|Maximum rows per file before splitting (`0` = no split). -|`zipped`|Wrap the output file in a zip archive. -|`doNotOpenNewFileInit`|Defer file creation until the first input row is received. -|`doNotCreateEmptyFile`|Delete the file at the end of the run if no rows were written. +|Compact|Suppress whitespace and EOL between elements; useful for byte-size-sensitive output. +|Blank line after XML declaration|Add a blank line right after the `` declaration. +|Emit empty elements|Emit an open/close tag pair for an element that has no value and no children. +|Emit attribute when value is null|Emit an attribute even when its source value is `null`. +|Emit attribute when no field is mapped|Emit an attribute that has no mapped field, using its default value. +|Trim leading/trailing whitespace|Trim text values before emitting them. +|Default decimal separator|Default decimal separator for numeric values; per-node settings still take precedence. +|Default grouping separator|Default grouping separator for numeric values; per-node settings still take precedence. +|Generate sibling XSD file|Write a sibling `.xsd` schema next to each output file (or each split). The schema is derived from the configured XML tree and the upstream row metadata. +|DOCTYPE root element / system / public identifier|Emit a `` declaration between the XML declaration and the root element. +|XSL stylesheet href / type|Emit an `` processing instruction. Type defaults to `text/xsl` when blank. |=== -=== XML Tree +=== XML Tree tab + +The XML Tree tab is the visual designer for the output structure. The left pane lists the input fields received from the previous transform; the right pane is split between the target tree (top) and the property pane (bottom) for the currently-selected node. + +==== Working with the tree + +* Click *Get fields* to (re)load the input fields from the previous transform. +* Drag a field from the left pane and drop it onto an element in the tree. A new child element is created with that field name and `mappedField` pre-filled. +* Use the toolbar above the tree (or the right-click menu) to: +** *+ Element* / *+ Attribute* / *+ Fragment*: add a child node of the chosen kind under the selected element. +** *Delete*: remove the selected node and its descendants (the root cannot be deleted). +** *Up* / *Down*: reorder the selected node among its siblings. +** *Loop*: toggle the loop flag. Exactly one element in the tree must carry it; switching the loop on a different node automatically clears it elsewhere. +** *Group-by*: toggle the group-by flag on an ancestor of the loop element. +* Selecting a node populates the *Properties* form below the tree. Edits propagate to the model immediately. -The XML tree is a recursive `XmlNode` graph. Each node has the following properties: +==== Node properties [options="header"] |=== |Property|Description -|`name`|Local name of the element or attribute. -|`namespace`|Optional XML namespace URI (treated as default namespace at that element). -|`kind`|`Element`, `Attribute`, or `DocumentFragment` (the latter parses the source field's value and inserts it as XML nodes rather than escaped text). -|`mappedField`|Input field whose value provides this node's content. For attributes and elements it sets the value; for nodes flagged `groupBy`, it identifies the group key. -|`defaultValue`|Static text used when `mappedField` is empty (or its value is `null`). -|`format`, `length`, `precision`, `currencySymbol`, `decimalSymbol`, `groupingSymbol`|Optional value-meta overrides used when converting the field value to a string. -|`forceCreate`|Output this node even when the value is `null`. -|`loop`|Marks this element as the row-loop element. Exactly one element in the tree must be flagged. -|`groupBy`|Marks this element as a group-by ancestor of the loop. Consecutive rows with equal `mappedField` values share a single occurrence. -|`children`|Recursive list of child `XmlNode`s. +|Name|Local name of the element or attribute. +|Namespace URI|Optional XML namespace URI. When set on the root element, it becomes the default namespace and is also written into the generated XSD as the `targetNamespace`. +|Kind|`Element`, `Attribute`, or `DocumentFragment`. The latter parses the source field's value and inserts it as XML nodes rather than escaped text. +|Mapped field|Input field whose value provides this node's content. For attributes and elements it sets the value; for nodes flagged `Group-by`, it identifies the group key only. +|Default value|Static text used when `Mapped field` is empty (or its value is `null`). +|Format / Length / Precision / Currency / Decimal / Grouping|Per-node value-meta overrides used when converting the field value to a string. Per-node settings take precedence over the global *Default decimal/grouping separator*. +|Loop|Marks this element as the row-loop element. Exactly one element must carry the flag. +|Group-by|Marks this element as a group-by ancestor of the loop. Consecutive rows with equal `Mapped field` values share a single occurrence. +|Force create|Output this node even when the value is `null` (uses the default value when set). |=== == Group-by behaviour -For the group-by mechanism to collapse correctly, *the input rows must already be sorted by the group-by key(s)*. Use a Sort Rows transform upstream if needed. When the key changes, the open group element is closed and a new one is opened. +For the group-by mechanism to collapse correctly, *the input rows must already be sorted by the group-by key(s)*. Use a Sort Rows transform upstream if needed. When the key changes, the open group element is closed and a new one is opened with the new key. + +== XSD generation + +When *Generate sibling XSD file* is enabled, the transform writes a `.xsd` schema next to each output file (or split). The schema: + +* declares one global element matching the root of the configured tree; +* nests complex types corresponding to elements with children or attributes; +* sets `maxOccurs="unbounded"` on the loop element and on every group-by ancestor; +* renders attributes as `xs:attribute` declarations (with `use="required"` when the source node is `Force create`); +* renders document-fragment nodes as `` placeholders; +* maps Hop value types to XSD built-ins as follows: integer → `xs:long`, number/big-number → `xs:decimal`, date/timestamp → `xs:dateTime`, boolean → `xs:boolean`, binary → `xs:base64Binary`, everything else → `xs:string`; +* uses the root node's namespace as the schema's `targetNamespace` (and `elementFormDefault="qualified"`) when set. + +The XSD is written outside zip archives and is added to the pipeline's result file list when *Add filename to result* is enabled. == Memory profile The transform uses StAX streaming and only buffers the XML state of the currently-open path of group elements. A single very large group is therefore O(largest group) in memory rather than O(document). + +== Example: orders with grouped items + +Input rows (already sorted by `orderId`): + +[options="header"] +|=== +|orderId|itemName|price +|1|foo|1.50 +|1|bar|2.00 +|2|baz|3.25 +|=== + +Tree: + +* `orders` (root, element) +** `order` (element, group-by, mapped field = `orderId`) +*** `id` (attribute, mapped field = `orderId`) +*** `item` (element, **loop**) +**** `name` (element, mapped field = `itemName`) +**** `price` (element, mapped field = `price`, format = `0.00`) + +Output: + +[source,xml] +---- + + + + foo1.50 + bar2.00 + + + baz3.25 + + +---- diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java index c125d0a5c0..645dc2febe 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -400,6 +400,7 @@ private void closeFile() { if (!data.fileOpen) { return; } + boolean rowsWritten = data.rowsWrittenToCurrentFile; try { data.writer.writeEndDocument(); data.writer.close(); @@ -418,6 +419,39 @@ private void closeFile() { } finally { data.fileOpen = false; } + if (rowsWritten && meta.isGenerateXsd()) { + writeSiblingXsd(); + } + } + + /** + * Writes a sibling .xsd schema for the current data file. Errors are logged but don't fail the + * pipeline (the data file is the contract; the schema is best-effort metadata). + */ + private void writeSiblingXsd() { + try { + String xsdName = meta.getFileSupport().buildXsdFilename(this, getCopy(), data.splitnr - 1); + AdvancedXmlOutputXsdWriter.write( + xsdName, this, meta.getEncoding(), meta.getRootNode(), data.inputRowMeta); + if (meta.getFileSupport().isAddToResultFilenames()) { + registerXsdResultFile(xsdName); + } + } catch (Exception e) { + logError("Error writing sibling XSD schema: " + e.getMessage(), e); + } + } + + private void registerXsdResultFile(String xsdName) { + try { + FileObject xsd = HopVfs.getFileObject(xsdName, this); + ResultFile rf = + new ResultFile( + ResultFile.FILE_TYPE_GENERAL, xsd, getPipelineMeta().getName(), getTransformName()); + rf.setComment("XSD schema generated by Advanced XML Output transform"); + addResultFile(rf); + } catch (Exception e) { + logError("Could not register sibling XSD as result file: " + e.getMessage()); + } } /** Discards a file that was opened but received no rows. */ @@ -642,13 +676,21 @@ private void writeElement(XmlNode node, Object[] r) throws Exception { data.writer.writeEndElement(); } + /** + * Writes the start tag for {@code node}, declaring its namespace (if any) as the default + * namespace on the element. The URI must be bound before the {@code writeStartElement(uri, ...)} + * call: woodstox (and other strict StAX impls) refuse to emit unbound namespace URIs. Children + * with no explicit namespace then inherit the default namespace of their nearest ancestor. + */ private void writeStartElementWithNamespace(XmlNode node) throws Exception { - if (!Utils.isEmpty(node.getNamespace())) { - data.writer.writeStartElement(node.getNamespace(), node.getName()); - data.writer.writeDefaultNamespace(node.getNamespace()); - } else { + String ns = node.getNamespace(); + if (Utils.isEmpty(ns)) { data.writer.writeStartElement(node.getName()); + return; } + data.writer.setDefaultNamespace(ns); + data.writer.writeStartElement(ns, node.getName()); + data.writer.writeDefaultNamespace(ns); } /** Writes any direct attribute children of {@code node} for the current row. */ diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index c91f1d22c5..bfbb6bfbb7 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -17,30 +17,47 @@ package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; import org.apache.hop.core.Const; +import org.apache.hop.core.Props; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; import org.apache.hop.ui.core.widget.TextVar; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; import org.eclipse.swt.widgets.Button; import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Display; import org.eclipse.swt.widgets.Label; import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Text; /** - * Phase-1 stub dialog for the Advanced XML Output transform. Exposes only the filename and encoding - * so the transform is editable in HopGui while the full tree designer (Phase 2) is implemented. The - * XML tree itself is configured programmatically or by editing the metadata directly until the - * designer ships. + * Dialog for the Advanced XML Output transform. + * + *

Three tabs: File (filename, encoding, splitting, zipping), Content (XML + * declaration, formatting, doctype, xsl, xsd) and XML Tree (visual designer for the + * hierarchical output structure with drag-and-drop). */ public class AdvancedXmlOutputDialog extends BaseTransformDialog { @@ -48,10 +65,43 @@ public class AdvancedXmlOutputDialog extends BaseTransformDialog { private final AdvancedXmlOutputMeta input; + // File tab private TextVar wFilename; + private Button wbFilename; private TextVar wExtension; - private TextVar wEncoding; + private Button wAddTransformnr; + private Button wAddDate; + private Button wAddTime; + private Button wSpecifyFormat; + private TextVar wDateTimeFormat; + private Text wSplitEvery; + private Button wZipped; + private Button wDoNotOpenAtInit; + private Button wDoNotCreateEmptyFile; private Button wAddToResult; + private CCombo wEncoding; + + // Content tab + private Button wCompactFile; + private Button wBlankLineAfterDecl; + private Button wCreateEmptyElement; + private Button wCreateAttributeIfNull; + private Button wCreateAttributeIfUnmapped; + private Button wTrim; + private TextVar wDefaultDecimal; + private TextVar wDefaultGroup; + private Button wGenerateXsd; + private TextVar wDoctypeRoot; + private TextVar wDoctypeSystem; + private TextVar wDoctypePublic; + private TextVar wXslHref; + private TextVar wXslType; + + // Tree designer tab + private XmlTreeDesigner wTreeDesigner; + + private final List inputFieldNames = new ArrayList<>(); + private boolean encodingsLoaded = false; public AdvancedXmlOutputDialog( Shell parent, @@ -64,127 +114,655 @@ public AdvancedXmlOutputDialog( @Override public String open() { - createShell(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.DialogTitle")); - - buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + Shell parent = getParent(); + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); + PropsUi.setLook(shell); + setShellImage(shell, input); ModifyListener lsMod = e -> input.setChanged(); changed = input.hasChanged(); - Composite wMain = new Composite(shell, SWT.NONE); - PropsUi.setLook(wMain); - FormLayout layout = new FormLayout(); - layout.marginWidth = 3; - layout.marginHeight = 3; - wMain.setLayout(layout); - - // Filename - Label wlFilename = new Label(wMain, SWT.RIGHT); - wlFilename.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Filename.Label")); - PropsUi.setLook(wlFilename); - FormData fdlFilename = new FormData(); - fdlFilename.left = new FormAttachment(0, 0); - fdlFilename.top = new FormAttachment(0, margin); - fdlFilename.right = new FormAttachment(middle, -margin); - wlFilename.setLayoutData(fdlFilename); - wFilename = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Shell.Title")); + + int margin = PropsUi.getMargin(); + int middle = props.getMiddlePct(); + + // Transform name + Label wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText( + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.TransformName.Label")); + PropsUi.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.top = new FormAttachment(0, margin); + fdlTransformName.right = new FormAttachment(middle, -margin); + wlTransformName.setLayoutData(fdlTransformName); + + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + PropsUi.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + // Tabs + CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); + + addFileTab(wTabFolder, lsMod, margin, middle); + addContentTab(wTabFolder, lsMod, margin, middle); + addTreeTab(wTabFolder, margin); + + FormData fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment(0, 0); + fdTabFolder.top = new FormAttachment(wTransformName, margin); + fdTabFolder.right = new FormAttachment(100, 0); + fdTabFolder.bottom = new FormAttachment(100, -50); + wTabFolder.setLayoutData(fdTabFolder); + + // Buttons + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wOk.addListener(SWT.Selection, e -> ok()); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + wCancel.addListener(SWT.Selection, e -> cancel()); + + setButtonPositions(new Button[] {wOk, wCancel}, margin, null); + + wTabFolder.setSelection(0); + + populateInputFieldsAsync(); + + getData(); + setSpecifyFormatVisibility(); + + input.setChanged(changed); + + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + + return transformName; + } + + // --------------------------------------------------------------------------- + // File tab + // --------------------------------------------------------------------------- + + private void addFileTab(CTabFolder tabFolder, ModifyListener lsMod, int margin, int middle) { + CTabItem tab = new CTabItem(tabFolder, SWT.NONE); + tab.setFont(GuiResource(shell)); + tab.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.FileTab.Title")); + + Composite comp = new Composite(tabFolder, SWT.NONE); + PropsUi.setLook(comp); + FormLayout fl = new FormLayout(); + fl.marginWidth = 3; + fl.marginHeight = 3; + comp.setLayout(fl); + + // Filename row + Label lblFn = new Label(comp, SWT.RIGHT); + lblFn.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Filename.Label")); + PropsUi.setLook(lblFn); + FormData fdLblFn = new FormData(); + fdLblFn.left = new FormAttachment(0, 0); + fdLblFn.top = new FormAttachment(0, margin); + fdLblFn.right = new FormAttachment(middle, -margin); + lblFn.setLayoutData(fdLblFn); + + wbFilename = new Button(comp, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbFilename); + wbFilename.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + FormData fdBtn = new FormData(); + fdBtn.right = new FormAttachment(100, 0); + fdBtn.top = new FormAttachment(0, margin); + wbFilename.setLayoutData(fdBtn); + + wFilename = new TextVar(variables, comp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); PropsUi.setLook(wFilename); wFilename.addModifyListener(lsMod); - FormData fdFilename = new FormData(); - fdFilename.left = new FormAttachment(middle, 0); - fdFilename.top = new FormAttachment(0, margin); - fdFilename.right = new FormAttachment(100, 0); - wFilename.setLayoutData(fdFilename); - - // Extension - Label wlExt = new Label(wMain, SWT.RIGHT); - wlExt.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Extension.Label")); - PropsUi.setLook(wlExt); - FormData fdlExt = new FormData(); - fdlExt.left = new FormAttachment(0, 0); - fdlExt.top = new FormAttachment(wFilename, margin); - fdlExt.right = new FormAttachment(middle, -margin); - wlExt.setLayoutData(fdlExt); - wExtension = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - PropsUi.setLook(wExtension); - wExtension.addModifyListener(lsMod); - FormData fdExt = new FormData(); - fdExt.left = new FormAttachment(middle, 0); - fdExt.top = new FormAttachment(wFilename, margin); - fdExt.right = new FormAttachment(100, 0); - wExtension.setLayoutData(fdExt); - - // Encoding - Label wlEnc = new Label(wMain, SWT.RIGHT); - wlEnc.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Encoding.Label")); - PropsUi.setLook(wlEnc); - FormData fdlEnc = new FormData(); - fdlEnc.left = new FormAttachment(0, 0); - fdlEnc.top = new FormAttachment(wExtension, margin); - fdlEnc.right = new FormAttachment(middle, -margin); - wlEnc.setLayoutData(fdlEnc); - wEncoding = new TextVar(variables, wMain, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + FormData fdFn = new FormData(); + fdFn.left = new FormAttachment(middle, 0); + fdFn.top = new FormAttachment(0, margin); + fdFn.right = new FormAttachment(wbFilename, -margin); + wFilename.setLayoutData(fdFn); + + wbFilename.addListener( + SWT.Selection, + e -> + BaseDialog.presentFileDialog( + true, + shell, + wFilename, + variables, + new String[] {"*.xml", "*"}, + new String[] { + BaseMessages.getString(PKG, "System.FileType.XMLFiles"), + BaseMessages.getString(PKG, "System.FileType.AllFiles") + }, + true)); + + wExtension = + addLabeledTextVar( + comp, + wFilename, + "Extension", + "AdvancedXMLOutputDialog.Extension.Label", + lsMod, + middle, + margin); + + Label lblEnc = new Label(comp, SWT.RIGHT); + lblEnc.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Encoding.Label")); + PropsUi.setLook(lblEnc); + FormData fdLblEnc = new FormData(); + fdLblEnc.left = new FormAttachment(0, 0); + fdLblEnc.right = new FormAttachment(middle, -margin); + fdLblEnc.top = new FormAttachment(wExtension, margin); + lblEnc.setLayoutData(fdLblEnc); + wEncoding = new CCombo(comp, SWT.BORDER | SWT.READ_ONLY); PropsUi.setLook(wEncoding); wEncoding.addModifyListener(lsMod); FormData fdEnc = new FormData(); fdEnc.left = new FormAttachment(middle, 0); - fdEnc.top = new FormAttachment(wExtension, margin); fdEnc.right = new FormAttachment(100, 0); + fdEnc.top = new FormAttachment(wExtension, margin); wEncoding.setLayoutData(fdEnc); + wEncoding.addFocusListener( + new org.eclipse.swt.events.FocusAdapter() { + @Override + public void focusGained(org.eclipse.swt.events.FocusEvent e) { + ensureEncodingsLoaded(); + } + }); - // Add to result - Label wlAddToResult = new Label(wMain, SWT.RIGHT); - wlAddToResult.setText( - BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.AddFileToResult.Label")); - PropsUi.setLook(wlAddToResult); - FormData fdlA = new FormData(); - fdlA.left = new FormAttachment(0, 0); - fdlA.top = new FormAttachment(wEncoding, margin); - fdlA.right = new FormAttachment(middle, -margin); - wlAddToResult.setLayoutData(fdlA); - wAddToResult = new Button(wMain, SWT.CHECK); - PropsUi.setLook(wAddToResult); - FormData fdA = new FormData(); - fdA.left = new FormAttachment(middle, 0); - fdA.top = new FormAttachment(wlAddToResult, 0, SWT.CENTER); - fdA.right = new FormAttachment(100, 0); - wAddToResult.setLayoutData(fdA); - - // Notice - Label wlNotice = new Label(wMain, SWT.WRAP); - wlNotice.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Notice.Label")); - PropsUi.setLook(wlNotice); - FormData fdN = new FormData(); - fdN.left = new FormAttachment(0, 0); - fdN.right = new FormAttachment(100, 0); - fdN.top = new FormAttachment(wAddToResult, margin * 3); - wlNotice.setLayoutData(fdN); - - FormData fdMain = new FormData(); - fdMain.left = new FormAttachment(0, 0); - fdMain.top = new FormAttachment(wSpacer, margin); - fdMain.right = new FormAttachment(100, 0); - fdMain.bottom = new FormAttachment(100, -50); - wMain.setLayoutData(fdMain); + wAddTransformnr = addCheckbox(comp, wEncoding, "AddTransformnr", lsMod, middle, margin); + wAddDate = addCheckbox(comp, wAddTransformnr, "AddDate", lsMod, middle, margin); + wAddTime = addCheckbox(comp, wAddDate, "AddTime", lsMod, middle, margin); + wSpecifyFormat = addCheckbox(comp, wAddTime, "SpecifyFormat", lsMod, middle, margin); + wSpecifyFormat.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + setSpecifyFormatVisibility(); + } + }); - getData(); - input.setChanged(changed); - focusTransformName(); - BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + wDateTimeFormat = + addLabeledTextVar( + comp, + wSpecifyFormat, + "DateTimeFormat", + "AdvancedXMLOutputDialog.DateTimeFormat.Label", + lsMod, + middle, + margin); - return transformName; + Label lblSplit = new Label(comp, SWT.RIGHT); + lblSplit.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.SplitEvery.Label")); + PropsUi.setLook(lblSplit); + FormData fdLblSplit = new FormData(); + fdLblSplit.left = new FormAttachment(0, 0); + fdLblSplit.right = new FormAttachment(middle, -margin); + fdLblSplit.top = new FormAttachment(wDateTimeFormat, margin); + lblSplit.setLayoutData(fdLblSplit); + wSplitEvery = new Text(comp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wSplitEvery); + wSplitEvery.addModifyListener(lsMod); + FormData fdSplit = new FormData(); + fdSplit.left = new FormAttachment(middle, 0); + fdSplit.right = new FormAttachment(100, 0); + fdSplit.top = new FormAttachment(wDateTimeFormat, margin); + wSplitEvery.setLayoutData(fdSplit); + + wZipped = addCheckbox(comp, wSplitEvery, "Zipped", lsMod, middle, margin); + wDoNotOpenAtInit = addCheckbox(comp, wZipped, "DoNotOpenAtInit", lsMod, middle, margin); + wDoNotCreateEmptyFile = + addCheckbox(comp, wDoNotOpenAtInit, "DoNotCreateEmptyFile", lsMod, middle, margin); + wAddToResult = addCheckbox(comp, wDoNotCreateEmptyFile, "AddToResult", lsMod, middle, margin); + + Button wShowFiles = new Button(comp, SWT.PUSH); + wShowFiles.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ShowFiles.Button")); + PropsUi.setLook(wShowFiles); + FormData fdShow = new FormData(); + fdShow.left = new FormAttachment(middle, 0); + fdShow.top = new FormAttachment(wAddToResult, margin * 2); + wShowFiles.setLayoutData(fdShow); + wShowFiles.addListener(SWT.Selection, e -> showFilesPreview()); + + FormData fdComp = new FormData(); + fdComp.left = new FormAttachment(0, 0); + fdComp.top = new FormAttachment(0, 0); + fdComp.right = new FormAttachment(100, 0); + fdComp.bottom = new FormAttachment(100, 0); + comp.setLayoutData(fdComp); + + comp.layout(); + tab.setControl(comp); } - private void getData() { - wFilename.setText(Const.NVL(input.getFileSupport().getFileName(), "")); - wExtension.setText(Const.NVL(input.getFileSupport().getExtension(), "")); - wEncoding.setText(Const.NVL(input.getEncoding(), Const.UTF_8)); - wAddToResult.setSelection(input.getFileSupport().isAddToResultFilenames()); + /** Pops up a dialog with up to a handful of sample filenames built from the current settings. */ + private void showFilesPreview() { + XmlFileOutputSupport snapshot = new XmlFileOutputSupport(); + snapshot.setFileName(wFilename.getText()); + snapshot.setExtension(wExtension.getText()); + snapshot.setSplitEvery(parsePositiveInt(wSplitEvery.getText(), 0)); + snapshot.setTransformNrInFilename(wAddTransformnr.getSelection()); + snapshot.setDateInFilename(wAddDate.getSelection()); + snapshot.setTimeInFilename(wAddTime.getSelection()); + snapshot.setSpecifyFormat(wSpecifyFormat.getSelection()); + snapshot.setDateTimeFormat(wDateTimeFormat.getText()); + snapshot.setZipped(wZipped.getSelection()); + + String[] files = snapshot.previewFilenames(variables); + if (files == null || files.length == 0) { + org.apache.hop.ui.core.dialog.MessageBox box = + new org.apache.hop.ui.core.dialog.MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + box.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ShowFiles.Title")); + box.setMessage(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ShowFiles.Empty")); + box.open(); + return; + } + org.apache.hop.ui.core.dialog.EnterSelectionDialog d = + new org.apache.hop.ui.core.dialog.EnterSelectionDialog( + shell, + files, + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ShowFiles.Title"), + BaseMessages.getString( + PKG, "AdvancedXMLOutputDialog.ShowFiles.Message", String.valueOf(files.length))); + d.setViewOnly(); + d.open(); + } + + // --------------------------------------------------------------------------- + // Content tab + // --------------------------------------------------------------------------- + + private void addContentTab(CTabFolder tabFolder, ModifyListener lsMod, int margin, int middle) { + CTabItem tab = new CTabItem(tabFolder, SWT.NONE); + tab.setFont(GuiResource(shell)); + tab.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ContentTab.Title")); + + Composite comp = new Composite(tabFolder, SWT.NONE); + PropsUi.setLook(comp); + FormLayout fl = new FormLayout(); + fl.marginWidth = 3; + fl.marginHeight = 3; + comp.setLayout(fl); + + wCompactFile = addCheckbox(comp, null, "CompactFile", lsMod, middle, margin); + wBlankLineAfterDecl = + addCheckbox(comp, wCompactFile, "BlankLineAfterDecl", lsMod, middle, margin); + wCreateEmptyElement = + addCheckbox(comp, wBlankLineAfterDecl, "CreateEmptyElement", lsMod, middle, margin); + wCreateAttributeIfNull = + addCheckbox(comp, wCreateEmptyElement, "CreateAttributeIfNull", lsMod, middle, margin); + wCreateAttributeIfUnmapped = + addCheckbox( + comp, wCreateAttributeIfNull, "CreateAttributeIfUnmapped", lsMod, middle, margin); + wTrim = addCheckbox(comp, wCreateAttributeIfUnmapped, "Trim", lsMod, middle, margin); + + wDefaultDecimal = + addLabeledTextVar( + comp, + wTrim, + "DefaultDecimal", + "AdvancedXMLOutputDialog.DefaultDecimal.Label", + lsMod, + middle, + margin); + wDefaultGroup = + addLabeledTextVar( + comp, + wDefaultDecimal, + "DefaultGroup", + "AdvancedXMLOutputDialog.DefaultGroup.Label", + lsMod, + middle, + margin); + + wGenerateXsd = addCheckbox(comp, wDefaultGroup, "GenerateXsd", lsMod, middle, margin); + + wDoctypeRoot = + addLabeledTextVar( + comp, + wGenerateXsd, + "DoctypeRoot", + "AdvancedXMLOutputDialog.DoctypeRoot.Label", + lsMod, + middle, + margin); + wDoctypeSystem = + addLabeledTextVar( + comp, + wDoctypeRoot, + "DoctypeSystem", + "AdvancedXMLOutputDialog.DoctypeSystem.Label", + lsMod, + middle, + margin); + wDoctypePublic = + addLabeledTextVar( + comp, + wDoctypeSystem, + "DoctypePublic", + "AdvancedXMLOutputDialog.DoctypePublic.Label", + lsMod, + middle, + margin); + + wXslHref = + addLabeledTextVar( + comp, + wDoctypePublic, + "XslHref", + "AdvancedXMLOutputDialog.XslHref.Label", + lsMod, + middle, + margin); + wXslType = + addLabeledTextVar( + comp, + wXslHref, + "XslType", + "AdvancedXMLOutputDialog.XslType.Label", + lsMod, + middle, + margin); + + FormData fdComp = new FormData(); + fdComp.left = new FormAttachment(0, 0); + fdComp.top = new FormAttachment(0, 0); + fdComp.right = new FormAttachment(100, 0); + fdComp.bottom = new FormAttachment(100, 0); + comp.setLayoutData(fdComp); + + comp.layout(); + tab.setControl(comp); + } + + // --------------------------------------------------------------------------- + // Tree tab + // --------------------------------------------------------------------------- + + private void addTreeTab(CTabFolder tabFolder, int margin) { + CTabItem tab = new CTabItem(tabFolder, SWT.NONE); + tab.setFont(GuiResource(shell)); + tab.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.TreeTab.Title")); + + Composite comp = new Composite(tabFolder, SWT.NONE); + PropsUi.setLook(comp); + FormLayout fl = new FormLayout(); + fl.marginWidth = 3; + fl.marginHeight = 3; + comp.setLayout(fl); + + wTreeDesigner = new XmlTreeDesigner(comp, SWT.NONE, variables); + PropsUi.setLook(wTreeDesigner); + FormData fdTd = new FormData(); + fdTd.left = new FormAttachment(0, 0); + fdTd.top = new FormAttachment(0, 0); + fdTd.right = new FormAttachment(100, 0); + fdTd.bottom = new FormAttachment(100, 0); + wTreeDesigner.setLayoutData(fdTd); + + wTreeDesigner.setChangeListener(input::setChanged); + wTreeDesigner.setGetFieldsListener(e -> reloadInputFieldsBlocking()); + + FormData fdComp = new FormData(); + fdComp.left = new FormAttachment(0, 0); + fdComp.top = new FormAttachment(0, 0); + fdComp.right = new FormAttachment(100, 0); + fdComp.bottom = new FormAttachment(100, 0); + comp.setLayoutData(fdComp); + + comp.layout(); + tab.setControl(comp); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + /** + * Adds a "label : TextVar" row anchored below {@code below}. + * + * @param messageKey i18n key for the label, or {@code null} to skip translation lookup + */ + private TextVar addLabeledTextVar( + Composite parent, + Control below, + String idHint, + String messageKey, + ModifyListener lsMod, + int middle, + int margin) { + Label lbl = new Label(parent, SWT.RIGHT); + String labelKey = + messageKey != null ? messageKey : "AdvancedXMLOutputDialog." + idHint + ".Label"; + lbl.setText(BaseMessages.getString(PKG, labelKey)); + PropsUi.setLook(lbl); + FormData fdLbl = new FormData(); + fdLbl.left = new FormAttachment(0, 0); + fdLbl.right = new FormAttachment(middle, -margin); + fdLbl.top = below == null ? new FormAttachment(0, margin) : new FormAttachment(below, margin); + lbl.setLayoutData(fdLbl); + + TextVar tv = new TextVar(variables, parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(tv); + tv.addModifyListener(lsMod); + FormData fd = new FormData(); + fd.left = new FormAttachment(middle, 0); + fd.right = new FormAttachment(100, 0); + fd.top = below == null ? new FormAttachment(0, margin) : new FormAttachment(below, margin); + tv.setLayoutData(fd); + return tv; + } + + private Button addCheckbox( + Composite parent, + Control below, + String idHint, + ModifyListener lsMod, + int middle, + int margin) { + Label lbl = new Label(parent, SWT.RIGHT); + lbl.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog." + idHint + ".Label")); + PropsUi.setLook(lbl); + FormData fdLbl = new FormData(); + fdLbl.left = new FormAttachment(0, 0); + fdLbl.right = new FormAttachment(middle, -margin); + fdLbl.top = below == null ? new FormAttachment(0, margin) : new FormAttachment(below, margin); + lbl.setLayoutData(fdLbl); + + Button b = new Button(parent, SWT.CHECK); + PropsUi.setLook(b); + FormData fdB = new FormData(); + fdB.left = new FormAttachment(middle, 0); + fdB.right = new FormAttachment(100, 0); + fdB.top = below == null ? new FormAttachment(0, margin) : new FormAttachment(below, margin); + b.setLayoutData(fdB); + b.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + if (lsMod != null) { + lsMod.modifyText(null); + } + } + }); + return b; + } + + /** Workaround to keep tab fonts consistent across the dialog. */ + private static org.eclipse.swt.graphics.Font GuiResource(Shell shell) { + return shell.getFont(); + } + + // --------------------------------------------------------------------------- + // Specify-format <-> add-date/time conditional logic + // --------------------------------------------------------------------------- + + private void setSpecifyFormatVisibility() { + boolean specify = wSpecifyFormat != null && wSpecifyFormat.getSelection(); + if (wDateTimeFormat != null) { + wDateTimeFormat.setEnabled(specify); + } + if (wAddDate != null) { + wAddDate.setEnabled(!specify); + } + if (wAddTime != null) { + wAddTime.setEnabled(!specify); + } + } + + private void ensureEncodingsLoaded() { + if (encodingsLoaded || wEncoding.isDisposed()) { + return; + } + encodingsLoaded = true; + List encs = new ArrayList<>(Charset.availableCharsets().keySet()); + java.util.Collections.sort(encs); + String current = wEncoding.getText(); + for (String e : encs) { + wEncoding.add(e); + } + if (!Utils.isEmpty(current)) { + int idx = wEncoding.indexOf(current); + if (idx >= 0) { + wEncoding.select(idx); + } else { + wEncoding.setText(current); + } + } + } + + // --------------------------------------------------------------------------- + // Input fields lookup (asynchronous, like the existing transform dialogs) + // --------------------------------------------------------------------------- + + private void populateInputFieldsAsync() { + Runnable r = + () -> { + TransformMeta tm = pipelineMeta.findTransform(transformName); + if (tm == null) { + return; + } + try { + IRowMeta row = pipelineMeta.getPrevTransformFields(variables, tm); + inputFieldNames.clear(); + if (row != null) { + for (int i = 0; i < row.size(); i++) { + inputFieldNames.add(row.getValueMeta(i).getName()); + } + } + Display.getDefault().asyncExec(this::pushInputFieldsToDesigner); + } catch (HopException e) { + logError(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ErrorGettingFields"), e); + } + }; + new Thread(r, "AdvancedXMLOutput-FieldLookup").start(); + } + + /** Synchronous re-fetch triggered by the "Get fields" button. */ + private void reloadInputFieldsBlocking() { + TransformMeta tm = pipelineMeta.findTransform(transformName); + if (tm == null) { + return; + } + try { + IRowMeta row = pipelineMeta.getPrevTransformFields(variables, tm); + inputFieldNames.clear(); + if (row != null) { + for (int i = 0; i < row.size(); i++) { + inputFieldNames.add(row.getValueMeta(i).getName()); + } + } + pushInputFieldsToDesigner(); + } catch (HopException e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ErrorGettingFields.Title"), + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ErrorGettingFields"), + e); + } + } + + private void pushInputFieldsToDesigner() { + if (wTreeDesigner != null && !wTreeDesigner.isDisposed()) { + wTreeDesigner.setInputFields(new ArrayList<>(inputFieldNames)); + } + } + + // --------------------------------------------------------------------------- + // Meta <-> dialog data binding + // --------------------------------------------------------------------------- + + /** Copies the data from the {@link AdvancedXmlOutputMeta} into the dialog widgets. */ + public void getData() { + XmlFileOutputSupport f = input.getFileSupport(); + if (f == null) { + f = new XmlFileOutputSupport(); + input.setFileSupport(f); + } + wFilename.setText(Const.NVL(f.getFileName(), "")); + wExtension.setText(Const.NVL(f.getExtension(), "xml")); + wAddTransformnr.setSelection(f.isTransformNrInFilename()); + wAddDate.setSelection(f.isDateInFilename()); + wAddTime.setSelection(f.isTimeInFilename()); + wSpecifyFormat.setSelection(f.isSpecifyFormat()); + wDateTimeFormat.setText(Const.NVL(f.getDateTimeFormat(), "")); + wSplitEvery.setText(f.getSplitEvery() > 0 ? Integer.toString(f.getSplitEvery()) : ""); + wZipped.setSelection(f.isZipped()); + wDoNotOpenAtInit.setSelection(f.isDoNotOpenNewFileInit()); + wDoNotCreateEmptyFile.setSelection(f.isDoNotCreateEmptyFile()); + wAddToResult.setSelection(f.isAddToResultFilenames()); + + String enc = Const.NVL(input.getEncoding(), "UTF-8"); + wEncoding.setText(enc); + + wCompactFile.setSelection(input.isCompactFile()); + wBlankLineAfterDecl.setSelection(input.isBlankLineAfterXmlDeclaration()); + wCreateEmptyElement.setSelection(input.isCreateEmptyElement()); + wCreateAttributeIfNull.setSelection(input.isCreateAttributeIfNull()); + wCreateAttributeIfUnmapped.setSelection(input.isCreateAttributeIfUnmapped()); + wTrim.setSelection(input.isTrimValues()); + wDefaultDecimal.setText(Const.NVL(input.getDefaultDecimalSeparator(), "")); + wDefaultGroup.setText(Const.NVL(input.getDefaultGroupingSeparator(), "")); + wGenerateXsd.setSelection(input.isGenerateXsd()); + wDoctypeRoot.setText(Const.NVL(input.getDoctypeRootElement(), "")); + wDoctypeSystem.setText(Const.NVL(input.getDoctypeSystemId(), "")); + wDoctypePublic.setText(Const.NVL(input.getDoctypePublicId(), "")); + wXslHref.setText(Const.NVL(input.getXslStylesheetHref(), "")); + wXslType.setText(Const.NVL(input.getXslStylesheetType(), "")); + + XmlNode root = + input.getRootNode() != null ? new XmlNode(input.getRootNode()) : defaultRootNode(); + wTreeDesigner.setRootNode(root); + + wTransformName.selectAll(); + wTransformName.setFocus(); + } + + private static XmlNode defaultRootNode() { + XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); + XmlNode loop = new XmlNode("Row", XmlNode.NodeKind.Element); + loop.setLoop(true); + root.addChild(loop); + return root; } private void cancel() { transformName = null; - input.setChanged(backupChanged); + input.setChanged(changed); dispose(); } @@ -193,10 +771,57 @@ private void ok() { return; } transformName = wTransformName.getText(); - input.getFileSupport().setFileName(wFilename.getText()); - input.getFileSupport().setExtension(wExtension.getText()); + + XmlFileOutputSupport f = input.getFileSupport(); + if (f == null) { + f = new XmlFileOutputSupport(); + input.setFileSupport(f); + } + f.setFileName(wFilename.getText()); + f.setExtension(wExtension.getText()); + f.setTransformNrInFilename(wAddTransformnr.getSelection()); + f.setDateInFilename(wAddDate.getSelection()); + f.setTimeInFilename(wAddTime.getSelection()); + f.setSpecifyFormat(wSpecifyFormat.getSelection()); + f.setDateTimeFormat(wDateTimeFormat.getText()); + f.setSplitEvery(parsePositiveInt(wSplitEvery.getText(), 0)); + f.setZipped(wZipped.getSelection()); + f.setDoNotOpenNewFileInit(wDoNotOpenAtInit.getSelection()); + f.setDoNotCreateEmptyFile(wDoNotCreateEmptyFile.getSelection()); + f.setAddToResultFilenames(wAddToResult.getSelection()); + input.setEncoding(wEncoding.getText()); - input.getFileSupport().setAddToResultFilenames(wAddToResult.getSelection()); + input.setCompactFile(wCompactFile.getSelection()); + input.setBlankLineAfterXmlDeclaration(wBlankLineAfterDecl.getSelection()); + input.setCreateEmptyElement(wCreateEmptyElement.getSelection()); + input.setCreateAttributeIfNull(wCreateAttributeIfNull.getSelection()); + input.setCreateAttributeIfUnmapped(wCreateAttributeIfUnmapped.getSelection()); + input.setTrimValues(wTrim.getSelection()); + input.setDefaultDecimalSeparator(wDefaultDecimal.getText()); + input.setDefaultGroupingSeparator(wDefaultGroup.getText()); + input.setGenerateXsd(wGenerateXsd.getSelection()); + input.setDoctypeRootElement(wDoctypeRoot.getText()); + input.setDoctypeSystemId(wDoctypeSystem.getText()); + input.setDoctypePublicId(wDoctypePublic.getText()); + input.setXslStylesheetHref(wXslHref.getText()); + input.setXslStylesheetType(wXslType.getText()); + + XmlNode designed = wTreeDesigner.getRootNode(); + input.setRootNode(designed != null ? new XmlNode(designed) : null); + + input.setChanged(); dispose(); } + + private static int parsePositiveInt(String s, int dflt) { + if (s == null || s.isBlank()) { + return dflt; + } + try { + int v = Integer.parseInt(s.trim()); + return v < 0 ? dflt : v; + } catch (NumberFormatException e) { + return dflt; + } + } } diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java index d80efc93c5..c28e078edb 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -128,7 +128,7 @@ public class AdvancedXmlOutputMeta injectionKeyDescription = "AdvancedXMLOutput.Injection.DEFAULT_GROUPING_SEPARATOR") private String defaultGroupingSeparator; - /** When true and any node declares a namespace, write a sibling .xsd next to the output file. */ + /** When true, write a sibling {@code .xsd} schema describing the produced XML structure. */ @HopMetadataProperty( key = "generate_xsd", injectionKey = "GENERATE_XSD", diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriter.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriter.java new file mode 100644 index 0000000000..6cf028c458 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriter.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamWriter; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; + +/** + * Generates a sibling XSD schema document from an {@link XmlNode} tree and the input row metadata. + * + *

The generated schema is intentionally lean (one global element, nested complex types) so it + * matches the StAX-streamed output shape exactly. Type information is derived from the input + * row-meta of every node that carries a {@code mappedField}; nodes without a mapping fall back to + * {@code xs:string}. + * + *

Multiplicity is set to {@code maxOccurs="unbounded"} for the loop element and any group-by + * ancestors. Document-fragment nodes are emitted as {@code } placeholders. + */ +final class AdvancedXmlOutputXsdWriter { + + private static final String XSD_NS = "http://www.w3.org/2001/XMLSchema"; + private static final String XSD_PREFIX = "xs"; + private static final XMLOutputFactory FACTORY = XMLOutputFactory.newInstance(); + + private AdvancedXmlOutputXsdWriter() {} + + /** + * Writes the XSD for {@code root} into the file located at {@code xsdFilename}. + * + * @param xsdFilename fully-resolved physical path of the .xsd file to write + * @param variables variable resolver used to materialize the {@link FileObject} + * @param encoding output character encoding (UTF-8 if blank) + * @param root root of the XML tree being described + * @param rowMeta input row metadata used to map mapped fields to XSD types (may be {@code null}) + */ + static void write( + String xsdFilename, IVariables variables, String encoding, XmlNode root, IRowMeta rowMeta) + throws Exception { + if (root == null) { + return; + } + FileObject fo = HopVfs.getFileObject(xsdFilename, variables); + try (OutputStream os = HopVfs.getOutputStream(fo, false)) { + String enc = Utils.isEmpty(encoding) ? "UTF-8" : encoding; + XMLStreamWriter w = FACTORY.createXMLStreamWriter(os, enc); + w.writeStartDocument(enc, "1.0"); + w.writeCharacters("\n"); + + String tns = root.getNamespace(); + w.writeStartElement(XSD_PREFIX, "schema", XSD_NS); + w.writeNamespace(XSD_PREFIX, XSD_NS); + if (!Utils.isEmpty(tns)) { + w.writeAttribute("targetNamespace", tns); + w.writeAttribute("elementFormDefault", "qualified"); + w.writeDefaultNamespace(tns); + } + + writeElement(w, root, rowMeta, true); + + w.writeEndElement(); // schema + w.writeEndDocument(); + w.close(); + } + } + + private static void writeElement( + XMLStreamWriter w, XmlNode node, IRowMeta rowMeta, boolean isRoot) throws Exception { + + boolean hasElementChildren = node.hasElementChildren(); + boolean hasAttributeChildren = node.hasAttributeChildren(); + boolean isLeaf = !hasElementChildren && !hasAttributeChildren; + + w.writeStartElement(XSD_PREFIX, "element", XSD_NS); + w.writeAttribute("name", Utils.isEmpty(node.getName()) ? "_" : node.getName()); + + if (!isRoot && (node.isLoop() || node.isGroupBy())) { + w.writeAttribute("minOccurs", "0"); + w.writeAttribute("maxOccurs", "unbounded"); + } + + if (isLeaf) { + w.writeAttribute("type", XSD_PREFIX + ":" + xsdSimpleTypeFor(node, rowMeta)); + w.writeEndElement(); + return; + } + + w.writeStartElement(XSD_PREFIX, "complexType", XSD_NS); + + List elements = childrenOfKind(node, XmlNode.NodeKind.Element); + List fragments = childrenOfKind(node, XmlNode.NodeKind.DocumentFragment); + List attributes = childrenOfKind(node, XmlNode.NodeKind.Attribute); + + boolean hasOwnText = + !Utils.isEmpty(node.getMappedField()) || !Utils.isEmpty(node.getDefaultValue()); + if (hasOwnText && !node.isGroupBy()) { + w.writeAttribute("mixed", "true"); + } + + if (!elements.isEmpty() || !fragments.isEmpty()) { + w.writeStartElement(XSD_PREFIX, "sequence", XSD_NS); + for (XmlNode c : elements) { + writeElement(w, c, rowMeta, false); + } + for (XmlNode c : fragments) { + w.writeStartElement(XSD_PREFIX, "any", XSD_NS); + w.writeAttribute("processContents", "skip"); + if (c.isLoop() || c.isGroupBy()) { + w.writeAttribute("minOccurs", "0"); + w.writeAttribute("maxOccurs", "unbounded"); + } else { + w.writeAttribute("minOccurs", "0"); + } + w.writeEndElement(); + } + w.writeEndElement(); // sequence + } + + for (XmlNode a : attributes) { + w.writeStartElement(XSD_PREFIX, "attribute", XSD_NS); + w.writeAttribute("name", Utils.isEmpty(a.getName()) ? "_" : a.getName()); + w.writeAttribute("type", XSD_PREFIX + ":" + xsdSimpleTypeFor(a, rowMeta)); + if (a.isForceCreate()) { + w.writeAttribute("use", "required"); + } else { + w.writeAttribute("use", "optional"); + } + w.writeEndElement(); + } + + w.writeEndElement(); // complexType + w.writeEndElement(); // element + } + + private static List childrenOfKind(XmlNode parent, XmlNode.NodeKind kind) { + List out = new ArrayList<>(); + if (parent.getChildren() != null) { + for (XmlNode c : parent.getChildren()) { + if (c.getKind() == kind) { + out.add(c); + } + } + } + return out; + } + + /** Maps a node's mapped field type to a built-in XSD simple type. Defaults to {@code string}. */ + static String xsdSimpleTypeFor(XmlNode node, IRowMeta rowMeta) { + String field = node.getMappedField(); + if (Utils.isEmpty(field) || rowMeta == null) { + return "string"; + } + int idx = rowMeta.indexOfValue(field); + if (idx < 0) { + return "string"; + } + IValueMeta vm = rowMeta.getValueMeta(idx); + return switch (vm.getType()) { + case IValueMeta.TYPE_INTEGER -> "long"; + case IValueMeta.TYPE_NUMBER, IValueMeta.TYPE_BIGNUMBER -> "decimal"; + case IValueMeta.TYPE_DATE, IValueMeta.TYPE_TIMESTAMP -> "dateTime"; + case IValueMeta.TYPE_BOOLEAN -> "boolean"; + case IValueMeta.TYPE_BINARY -> "base64Binary"; + default -> "string"; + }; + } +} diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java index ebd56c3138..337a4e8255 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java @@ -191,6 +191,24 @@ public String buildFilename(IVariables variables, int copyNr, int splitNr, boole return filename; } + /** + * Returns the sibling XSD filename for the data file at {@code copyNr}/{@code splitNr}. Always + * uses extension {@code xsd} regardless of the data file's extension and is never wrapped in a + * zip archive (the XSD lives next to the archive, not inside it). + */ + public String buildXsdFilename(IVariables variables, int copyNr, int splitNr) { + String saveExt = extension; + boolean saveZipped = zipped; + try { + this.zipped = false; + this.extension = "xsd"; + return buildFilename(variables, copyNr, splitNr, false); + } finally { + this.extension = saveExt; + this.zipped = saveZipped; + } + } + /** Returns up to {@code nr} sample filenames for use in the dialog's "Show files" preview. */ public String[] previewFilenames(IVariables variables) { int copies = transformNrInFilename ? 3 : 1; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java new file mode 100644 index 0000000000..d80a3fc113 --- /dev/null +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java @@ -0,0 +1,939 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; +import org.apache.hop.core.Const; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.ui.core.PropsUi; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.SashForm; +import org.eclipse.swt.dnd.DND; +import org.eclipse.swt.dnd.DragSource; +import org.eclipse.swt.dnd.DragSourceEvent; +import org.eclipse.swt.dnd.DragSourceListener; +import org.eclipse.swt.dnd.DropTarget; +import org.eclipse.swt.dnd.DropTargetEvent; +import org.eclipse.swt.dnd.DropTargetListener; +import org.eclipse.swt.dnd.TextTransfer; +import org.eclipse.swt.dnd.Transfer; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Listener; +import org.eclipse.swt.widgets.Menu; +import org.eclipse.swt.widgets.MenuItem; +import org.eclipse.swt.widgets.Text; +import org.eclipse.swt.widgets.ToolBar; +import org.eclipse.swt.widgets.ToolItem; +import org.eclipse.swt.widgets.Tree; +import org.eclipse.swt.widgets.TreeItem; + +/** + * SWT composite that lets the user design an {@link XmlNode} tree visually. + * + *

Layout: a horizontal sash with the input-fields list on the left (drag source) and a vertical + * sash on the right hosting the target tree (drop target + toolbar / context menu) on top and a + * properties form for the selected node on the bottom. + * + *

The composite owns the {@link XmlNode} model passed via {@link #setRootNode(XmlNode)} and + * mutates it in place; the host dialog reads the (mutated) model back via {@link #getRootNode()} + * when the user clicks OK. + */ +public class XmlTreeDesigner extends Composite { + + private static final Class PKG = AdvancedXmlOutputMeta.class; + + /** Listener that observers can register to be notified that the tree was modified. */ + public interface ChangeListener { + void onChanged(); + } + + private final IVariables variables; + + // Left side: input fields + private org.eclipse.swt.widgets.List wInputFields; + private Button wGetFields; + + // Right top: tree + toolbar + private Tree wTree; + + // Right bottom: properties form + private Text wpName; + private Text wpNamespace; + private CCombo wpKind; + private CCombo wpMappedField; + private Text wpDefaultValue; + private Text wpFormat; + private Text wpLength; + private Text wpPrecision; + private Text wpCurrency; + private Text wpDecimal; + private Text wpGroup; + private Button wpLoop; + private Button wpGroupBy; + private Button wpForceCreate; + + /** Root of the tree being designed; never {@code null} after {@link #setRootNode(XmlNode)}. */ + private XmlNode rootNode; + + /** Available input field names, used for the field combo and the drag-source list. */ + private final List inputFields = new ArrayList<>(); + + /** Avoid feedback loops when populating the properties form for the current selection. */ + private boolean updatingProperties = false; + + private ChangeListener changeListener; + + public XmlTreeDesigner(Composite parent, int style, IVariables variables) { + super(parent, style); + this.variables = variables; + setLayout(new FormLayout()); + build(); + } + + // --------------------------------------------------------------------------- + // Public API used by the host dialog + // --------------------------------------------------------------------------- + + public void setRootNode(XmlNode root) { + this.rootNode = root == null ? defaultRoot() : root; + refreshTree(); + if (wTree.getItemCount() > 0) { + wTree.setSelection(wTree.getItem(0)); + handleSelectionChanged(); + } + } + + public XmlNode getRootNode() { + return rootNode; + } + + public void setInputFields(List fields) { + this.inputFields.clear(); + if (fields != null) { + this.inputFields.addAll(fields); + } + refreshInputFieldsList(); + refreshMappedFieldCombo(); + } + + public void setChangeListener(ChangeListener listener) { + this.changeListener = listener; + } + + public void setGetFieldsListener(Listener listener) { + if (wGetFields != null && listener != null) { + wGetFields.addListener(SWT.Selection, listener); + } + } + + // --------------------------------------------------------------------------- + // UI build + // --------------------------------------------------------------------------- + + private void build() { + SashForm hSash = new SashForm(this, SWT.HORIZONTAL); + PropsUi.setLook(hSash); + FormData fdSash = new FormData(); + fdSash.left = new FormAttachment(0, 0); + fdSash.top = new FormAttachment(0, 0); + fdSash.right = new FormAttachment(100, 0); + fdSash.bottom = new FormAttachment(100, 0); + hSash.setLayoutData(fdSash); + + buildLeftPane(hSash); + buildRightPane(hSash); + hSash.setWeights(new int[] {25, 75}); + } + + private void buildLeftPane(Composite parent) { + Composite left = new Composite(parent, SWT.NONE); + PropsUi.setLook(left); + FormLayout fl = new FormLayout(); + fl.marginWidth = 3; + fl.marginHeight = 3; + left.setLayout(fl); + + Label lbl = new Label(left, SWT.NONE); + lbl.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.InputFields.Label")); + PropsUi.setLook(lbl); + FormData fdLbl = new FormData(); + fdLbl.left = new FormAttachment(0, 0); + fdLbl.top = new FormAttachment(0, 0); + fdLbl.right = new FormAttachment(100, 0); + lbl.setLayoutData(fdLbl); + + wGetFields = new Button(left, SWT.PUSH); + wGetFields.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.GetFields.Button")); + PropsUi.setLook(wGetFields); + FormData fdGet = new FormData(); + fdGet.left = new FormAttachment(0, 0); + fdGet.right = new FormAttachment(100, 0); + fdGet.bottom = new FormAttachment(100, 0); + wGetFields.setLayoutData(fdGet); + + wInputFields = + new org.eclipse.swt.widgets.List( + left, SWT.SINGLE | SWT.V_SCROLL | SWT.H_SCROLL | SWT.BORDER); + PropsUi.setLook(wInputFields); + FormData fdList = new FormData(); + fdList.left = new FormAttachment(0, 0); + fdList.right = new FormAttachment(100, 0); + fdList.top = new FormAttachment(lbl, 3); + fdList.bottom = new FormAttachment(wGetFields, -3); + wInputFields.setLayoutData(fdList); + + setupFieldsDragSource(); + } + + private void buildRightPane(Composite parent) { + SashForm vSash = new SashForm(parent, SWT.VERTICAL); + PropsUi.setLook(vSash); + buildTreeWithToolbar(vSash); + buildPropertiesPane(vSash); + vSash.setWeights(new int[] {65, 35}); + } + + private void buildTreeWithToolbar(Composite parent) { + Composite top = new Composite(parent, SWT.NONE); + PropsUi.setLook(top); + FormLayout fl = new FormLayout(); + fl.marginWidth = 3; + fl.marginHeight = 3; + top.setLayout(fl); + + ToolBar tb = new ToolBar(top, SWT.FLAT | SWT.HORIZONTAL); + PropsUi.setLook(tb); + FormData fdTb = new FormData(); + fdTb.left = new FormAttachment(0, 0); + fdTb.top = new FormAttachment(0, 0); + fdTb.right = new FormAttachment(100, 0); + tb.setLayoutData(fdTb); + + addToolItem(tb, "AddElement", e -> addChild(XmlNode.NodeKind.Element)); + addToolItem(tb, "AddAttribute", e -> addChild(XmlNode.NodeKind.Attribute)); + addToolItem(tb, "AddFragment", e -> addChild(XmlNode.NodeKind.DocumentFragment)); + new ToolItem(tb, SWT.SEPARATOR); + addToolItem(tb, "Delete", e -> deleteSelected()); + addToolItem(tb, "MoveUp", e -> moveSelected(-1)); + addToolItem(tb, "MoveDown", e -> moveSelected(+1)); + new ToolItem(tb, SWT.SEPARATOR); + addToolItem(tb, "ToggleLoop", e -> toggleSelectedFlag(true, false)); + addToolItem(tb, "ToggleGroupBy", e -> toggleSelectedFlag(false, true)); + + wTree = new Tree(top, SWT.BORDER | SWT.SINGLE | SWT.V_SCROLL | SWT.H_SCROLL); + PropsUi.setLook(wTree); + FormData fdTree = new FormData(); + fdTree.left = new FormAttachment(0, 0); + fdTree.top = new FormAttachment(tb, 3); + fdTree.right = new FormAttachment(100, 0); + fdTree.bottom = new FormAttachment(100, 0); + wTree.setLayoutData(fdTree); + + wTree.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + handleSelectionChanged(); + } + }); + + setupTreeDropTarget(); + setupTreeContextMenu(); + } + + private void addToolItem(ToolBar tb, String key, Listener selectionListener) { + ToolItem ti = new ToolItem(tb, SWT.PUSH); + ti.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Toolbar." + key)); + ti.setToolTipText( + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Toolbar." + key + ".Tooltip")); + ti.addListener(SWT.Selection, selectionListener); + } + + private void buildPropertiesPane(Composite parent) { + Composite props = new Composite(parent, SWT.NONE); + PropsUi.setLook(props); + FormLayout fl = new FormLayout(); + fl.marginWidth = 6; + fl.marginHeight = 6; + props.setLayout(fl); + + int labelWidth = 110; + + wpName = addLabeledText(props, null, "Name", labelWidth); + wpNamespace = addLabeledText(props, wpName, "Namespace", labelWidth); + + Label lblKind = addLabel(props, wpNamespace, "Kind", labelWidth); + wpKind = new CCombo(props, SWT.BORDER | SWT.READ_ONLY); + PropsUi.setLook(wpKind); + for (XmlNode.NodeKind k : XmlNode.NodeKind.values()) { + wpKind.add(k.name()); + } + layoutValueRight(lblKind, wpKind, labelWidth); + wpKind.addModifyListener(e -> applyPropertiesToModel()); + + Label lblField = addLabel(props, lblKind, "MappedField", labelWidth); + wpMappedField = new CCombo(props, SWT.BORDER); + PropsUi.setLook(wpMappedField); + layoutValueRight(lblField, wpMappedField, labelWidth); + wpMappedField.addModifyListener(e -> applyPropertiesToModel()); + + wpDefaultValue = addLabeledText(props, lblField, "DefaultValue", labelWidth); + wpFormat = addLabeledText(props, wpDefaultValue, "Format", labelWidth); + + Composite numericRow = new Composite(props, SWT.NONE); + PropsUi.setLook(numericRow); + FormLayout nrL = new FormLayout(); + nrL.marginWidth = 0; + nrL.marginHeight = 0; + numericRow.setLayout(nrL); + FormData fdNr = new FormData(); + fdNr.left = new FormAttachment(0, labelWidth); + fdNr.right = new FormAttachment(100, 0); + fdNr.top = new FormAttachment(wpFormat, 3); + numericRow.setLayoutData(fdNr); + + Label lblL = new Label(numericRow, SWT.NONE); + lblL.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Properties.Length")); + PropsUi.setLook(lblL); + FormData fdLblL = new FormData(); + fdLblL.left = new FormAttachment(0, 0); + fdLblL.top = new FormAttachment(0, 0); + lblL.setLayoutData(fdLblL); + + wpLength = new Text(numericRow, SWT.BORDER | SWT.SINGLE); + PropsUi.setLook(wpLength); + FormData fdLen = new FormData(); + fdLen.left = new FormAttachment(lblL, 3); + fdLen.top = new FormAttachment(0, 0); + fdLen.width = 60; + wpLength.setLayoutData(fdLen); + wpLength.addModifyListener(e -> applyPropertiesToModel()); + + Label lblP = new Label(numericRow, SWT.NONE); + lblP.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Properties.Precision")); + PropsUi.setLook(lblP); + FormData fdLblP = new FormData(); + fdLblP.left = new FormAttachment(wpLength, 12); + fdLblP.top = new FormAttachment(0, 0); + lblP.setLayoutData(fdLblP); + + wpPrecision = new Text(numericRow, SWT.BORDER | SWT.SINGLE); + PropsUi.setLook(wpPrecision); + FormData fdPrec = new FormData(); + fdPrec.left = new FormAttachment(lblP, 3); + fdPrec.top = new FormAttachment(0, 0); + fdPrec.width = 60; + wpPrecision.setLayoutData(fdPrec); + wpPrecision.addModifyListener(e -> applyPropertiesToModel()); + + wpCurrency = addLabeledText(props, numericRow, "Currency", labelWidth); + wpDecimal = addLabeledText(props, wpCurrency, "Decimal", labelWidth); + wpGroup = addLabeledText(props, wpDecimal, "Grouping", labelWidth); + + Composite flagsRow = new Composite(props, SWT.NONE); + PropsUi.setLook(flagsRow); + FormLayout flR = new FormLayout(); + flR.marginWidth = 0; + flR.marginHeight = 0; + flagsRow.setLayout(flR); + FormData fdFlR = new FormData(); + fdFlR.left = new FormAttachment(0, labelWidth); + fdFlR.right = new FormAttachment(100, 0); + fdFlR.top = new FormAttachment(wpGroup, 6); + flagsRow.setLayoutData(fdFlR); + + wpLoop = newFlagButton(flagsRow, null, "Loop"); + wpGroupBy = newFlagButton(flagsRow, wpLoop, "GroupBy"); + wpForceCreate = newFlagButton(flagsRow, wpGroupBy, "ForceCreate"); + } + + private Button newFlagButton(Composite parent, Button leftOf, String key) { + Button b = new Button(parent, SWT.CHECK); + b.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Properties." + key)); + PropsUi.setLook(b); + FormData fd = new FormData(); + fd.top = new FormAttachment(0, 0); + if (leftOf == null) { + fd.left = new FormAttachment(0, 0); + } else { + fd.left = new FormAttachment(leftOf, 12); + } + b.setLayoutData(fd); + b.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + applyPropertiesToModel(); + } + }); + return b; + } + + private Label addLabel(Composite parent, Control below, String key, int labelWidth) { + Label lbl = new Label(parent, SWT.NONE); + lbl.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Properties." + key)); + PropsUi.setLook(lbl); + FormData fd = new FormData(); + fd.left = new FormAttachment(0, 0); + fd.width = labelWidth - 3; + fd.top = below == null ? new FormAttachment(0, 0) : new FormAttachment(below, 6); + lbl.setLayoutData(fd); + return lbl; + } + + private void layoutValueRight(Control labelControl, Control valueControl, int labelWidth) { + FormData fd = new FormData(); + fd.left = new FormAttachment(0, labelWidth); + fd.right = new FormAttachment(100, 0); + fd.top = new FormAttachment(labelControl, 0, SWT.TOP); + valueControl.setLayoutData(fd); + } + + private Text addLabeledText(Composite parent, Control below, String key, int labelWidth) { + Label lbl = addLabel(parent, below, key, labelWidth); + Text t = new Text(parent, SWT.BORDER | SWT.SINGLE); + PropsUi.setLook(t); + layoutValueRight(lbl, t, labelWidth); + t.addModifyListener(e -> applyPropertiesToModel()); + return t; + } + + // --------------------------------------------------------------------------- + // DnD: dragging fields from the input list to the tree + // --------------------------------------------------------------------------- + + private void setupFieldsDragSource() { + Transfer[] transfers = new Transfer[] {TextTransfer.getInstance()}; + DragSource source = new DragSource(wInputFields, DND.DROP_COPY | DND.DROP_MOVE); + source.setTransfer(transfers); + source.addDragListener( + new DragSourceListener() { + @Override + public void dragStart(DragSourceEvent event) { + event.doit = wInputFields.getSelectionCount() > 0; + } + + @Override + public void dragSetData(DragSourceEvent event) { + String[] sel = wInputFields.getSelection(); + StringBuilder sb = new StringBuilder(); + for (String s : sel) { + sb.append(s).append(Const.CR); + } + event.data = sb.toString(); + } + + @Override + public void dragFinished(DragSourceEvent event) { + // nothing + } + }); + } + + private void setupTreeDropTarget() { + Transfer[] transfers = new Transfer[] {TextTransfer.getInstance()}; + DropTarget target = new DropTarget(wTree, DND.DROP_COPY | DND.DROP_MOVE); + target.setTransfer(transfers); + target.addDropListener( + new DropTargetListener() { + @Override + public void dragEnter(DropTargetEvent event) { + // nothing + } + + @Override + public void dragLeave(DropTargetEvent event) { + // nothing + } + + @Override + public void dragOperationChanged(DropTargetEvent event) { + // nothing + } + + @Override + public void dragOver(DropTargetEvent event) { + event.feedback = DND.FEEDBACK_SELECT | DND.FEEDBACK_EXPAND | DND.FEEDBACK_SCROLL; + } + + @Override + public void drop(DropTargetEvent event) { + if (event.data == null) { + event.detail = DND.DROP_NONE; + return; + } + XmlNode targetNode = + event.item instanceof TreeItem ti ? (XmlNode) ti.getData() : rootNode; + if (targetNode == null) { + return; + } + // Adding under an Attribute makes no sense - redirect to its parent. + if (targetNode.getKind() == XmlNode.NodeKind.Attribute) { + XmlNode parent = findParent(targetNode); + targetNode = parent != null ? parent : rootNode; + } + StringTokenizer tok = new StringTokenizer((String) event.data, Const.CR); + XmlNode lastAdded = null; + while (tok.hasMoreTokens()) { + String fieldName = tok.nextToken(); + if (Utils.isEmpty(fieldName)) { + continue; + } + XmlNode added = new XmlNode(fieldName, XmlNode.NodeKind.Element); + added.setMappedField(fieldName); + targetNode.addChild(added); + lastAdded = added; + } + refreshTree(); + if (lastAdded != null) { + selectNode(lastAdded); + } + fireChanged(); + } + + @Override + public void dropAccept(DropTargetEvent event) { + // nothing + } + }); + } + + // --------------------------------------------------------------------------- + // Context menu mirroring the toolbar + // --------------------------------------------------------------------------- + + private void setupTreeContextMenu() { + Menu menu = new Menu(wTree); + + menu.addListener( + SWT.Show, + e -> { + XmlNode sel = currentSelection(); + for (MenuItem mi : menu.getItems()) { + mi.dispose(); + } + buildMenu(menu, sel); + }); + + wTree.setMenu(menu); + } + + private void buildMenu(Menu menu, XmlNode sel) { + addMenu(menu, "AddElement", () -> addChild(XmlNode.NodeKind.Element)); + addMenu(menu, "AddAttribute", () -> addChild(XmlNode.NodeKind.Attribute)); + addMenu(menu, "AddFragment", () -> addChild(XmlNode.NodeKind.DocumentFragment)); + new MenuItem(menu, SWT.SEPARATOR); + addMenu(menu, "Delete", this::deleteSelected, sel != null && sel != rootNode); + addMenu(menu, "MoveUp", () -> moveSelected(-1), sel != null && sel != rootNode); + addMenu(menu, "MoveDown", () -> moveSelected(+1), sel != null && sel != rootNode); + new MenuItem(menu, SWT.SEPARATOR); + addCheckMenu( + menu, "ToggleLoop", sel != null && sel.isLoop(), () -> toggleSelectedFlag(true, false)); + addCheckMenu( + menu, + "ToggleGroupBy", + sel != null && sel.isGroupBy(), + () -> toggleSelectedFlag(false, true)); + } + + private void addMenu(Menu parent, String key, Runnable action) { + addMenu(parent, key, action, true); + } + + private void addMenu(Menu parent, String key, Runnable action, boolean enabled) { + MenuItem mi = new MenuItem(parent, SWT.PUSH); + mi.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Toolbar." + key)); + mi.setEnabled(enabled); + mi.addListener(SWT.Selection, ignore -> action.run()); + } + + private void addCheckMenu(Menu parent, String key, boolean selected, Runnable action) { + MenuItem mi = new MenuItem(parent, SWT.CHECK); + mi.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Toolbar." + key)); + mi.setSelection(selected); + mi.addListener(SWT.Selection, ignore -> action.run()); + } + + // --------------------------------------------------------------------------- + // Tree mutations + // --------------------------------------------------------------------------- + + private void addChild(XmlNode.NodeKind kind) { + XmlNode parent = currentSelection(); + if (parent == null) { + parent = rootNode; + } + if (parent.getKind() == XmlNode.NodeKind.Attribute + || parent.getKind() == XmlNode.NodeKind.DocumentFragment) { + // Re-parent to the actual parent element. + XmlNode p = findParent(parent); + parent = p != null ? p : rootNode; + } + XmlNode child = + new XmlNode(kind == XmlNode.NodeKind.Attribute ? "newAttr" : "newElement", kind); + parent.addChild(child); + refreshTree(); + selectNode(child); + fireChanged(); + } + + private void deleteSelected() { + XmlNode sel = currentSelection(); + if (sel == null || sel == rootNode) { + return; + } + XmlNode parent = findParent(sel); + if (parent != null && parent.getChildren() != null) { + parent.getChildren().remove(sel); + refreshTree(); + selectNode(parent); + fireChanged(); + } + } + + private void moveSelected(int delta) { + XmlNode sel = currentSelection(); + if (sel == null || sel == rootNode) { + return; + } + XmlNode parent = findParent(sel); + if (parent == null || parent.getChildren() == null) { + return; + } + List sibs = parent.getChildren(); + int idx = sibs.indexOf(sel); + int newIdx = idx + delta; + if (idx < 0 || newIdx < 0 || newIdx >= sibs.size()) { + return; + } + sibs.remove(idx); + sibs.add(newIdx, sel); + refreshTree(); + selectNode(sel); + fireChanged(); + } + + /** + * Toggle the loop / group-by flag of the current selection. + * + *

Loop is mutually exclusive across the entire tree; setting one node as loop clears the flag + * elsewhere first. Group-by may co-exist on multiple ancestors of the loop. + */ + private void toggleSelectedFlag(boolean toggleLoop, boolean toggleGroupBy) { + XmlNode sel = currentSelection(); + if (sel == null) { + return; + } + if (toggleLoop) { + boolean newVal = !sel.isLoop(); + if (newVal) { + clearLoopFlag(rootNode); + } + sel.setLoop(newVal); + } + if (toggleGroupBy) { + sel.setGroupBy(!sel.isGroupBy()); + } + refreshTree(); + selectNode(sel); + fireChanged(); + } + + private void clearLoopFlag(XmlNode n) { + n.setLoop(false); + if (n.getChildren() != null) { + for (XmlNode c : n.getChildren()) { + clearLoopFlag(c); + } + } + } + + // --------------------------------------------------------------------------- + // Properties form ↔ model sync + // --------------------------------------------------------------------------- + + private void handleSelectionChanged() { + XmlNode n = currentSelection(); + populateProperties(n); + } + + private void populateProperties(XmlNode n) { + updatingProperties = true; + try { + boolean enabled = n != null; + setPropertiesEnabled(enabled); + if (n == null) { + wpName.setText(""); + wpNamespace.setText(""); + wpKind.select(0); + wpMappedField.setText(""); + wpDefaultValue.setText(""); + wpFormat.setText(""); + wpLength.setText(""); + wpPrecision.setText(""); + wpCurrency.setText(""); + wpDecimal.setText(""); + wpGroup.setText(""); + wpLoop.setSelection(false); + wpGroupBy.setSelection(false); + wpForceCreate.setSelection(false); + return; + } + wpName.setText(Const.NVL(n.getName(), "")); + wpNamespace.setText(Const.NVL(n.getNamespace(), "")); + wpKind.setText(n.getKind() == null ? XmlNode.NodeKind.Element.name() : n.getKind().name()); + wpMappedField.setText(Const.NVL(n.getMappedField(), "")); + wpDefaultValue.setText(Const.NVL(n.getDefaultValue(), "")); + wpFormat.setText(Const.NVL(n.getFormat(), "")); + wpLength.setText(n.getLength() > 0 ? Integer.toString(n.getLength()) : ""); + wpPrecision.setText(n.getPrecision() > 0 ? Integer.toString(n.getPrecision()) : ""); + wpCurrency.setText(Const.NVL(n.getCurrencySymbol(), "")); + wpDecimal.setText(Const.NVL(n.getDecimalSymbol(), "")); + wpGroup.setText(Const.NVL(n.getGroupingSymbol(), "")); + wpLoop.setSelection(n.isLoop()); + wpGroupBy.setSelection(n.isGroupBy()); + wpForceCreate.setSelection(n.isForceCreate()); + } finally { + updatingProperties = false; + } + } + + private void setPropertiesEnabled(boolean enabled) { + wpName.setEnabled(enabled); + wpNamespace.setEnabled(enabled); + wpKind.setEnabled(enabled); + wpMappedField.setEnabled(enabled); + wpDefaultValue.setEnabled(enabled); + wpFormat.setEnabled(enabled); + wpLength.setEnabled(enabled); + wpPrecision.setEnabled(enabled); + wpCurrency.setEnabled(enabled); + wpDecimal.setEnabled(enabled); + wpGroup.setEnabled(enabled); + wpLoop.setEnabled(enabled); + wpGroupBy.setEnabled(enabled); + wpForceCreate.setEnabled(enabled); + } + + private void applyPropertiesToModel() { + if (updatingProperties) { + return; + } + XmlNode n = currentSelection(); + if (n == null) { + return; + } + n.setName(wpName.getText()); + n.setNamespace(wpNamespace.getText()); + n.setKind(XmlNode.NodeKind.getIfPresent(wpKind.getText())); + n.setMappedField(wpMappedField.getText()); + n.setDefaultValue(wpDefaultValue.getText()); + n.setFormat(wpFormat.getText()); + n.setLength(parseInt(wpLength.getText(), -1)); + n.setPrecision(parseInt(wpPrecision.getText(), -1)); + n.setCurrencySymbol(wpCurrency.getText()); + n.setDecimalSymbol(wpDecimal.getText()); + n.setGroupingSymbol(wpGroup.getText()); + + // Loop flag is mutually exclusive + boolean wantLoop = wpLoop.getSelection(); + if (wantLoop && !n.isLoop()) { + clearLoopFlag(rootNode); + } + n.setLoop(wantLoop); + n.setGroupBy(wpGroupBy.getSelection()); + n.setForceCreate(wpForceCreate.getSelection()); + + refreshSelectedTreeItemLabel(); + fireChanged(); + } + + private static int parseInt(String s, int dflt) { + if (s == null || s.isBlank()) { + return dflt; + } + try { + return Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + return dflt; + } + } + + // --------------------------------------------------------------------------- + // Tree refresh / lookup + // --------------------------------------------------------------------------- + + /** Rebuilds the tree from the model. Preserves expansion where possible. */ + private void refreshTree() { + wTree.removeAll(); + if (rootNode == null) { + return; + } + TreeItem rootItem = new TreeItem(wTree, SWT.NONE); + populateItem(rootItem, rootNode); + rootItem.setExpanded(true); + } + + private void populateItem(TreeItem item, XmlNode node) { + item.setData(node); + item.setText(formatNodeLabel(node)); + if (node.getChildren() != null) { + for (XmlNode c : node.getChildren()) { + TreeItem child = new TreeItem(item, SWT.NONE); + populateItem(child, c); + } + } + item.setExpanded(true); + } + + private static String formatNodeLabel(XmlNode n) { + StringBuilder sb = new StringBuilder(); + sb.append(n.getKind() == XmlNode.NodeKind.Attribute ? "@" : ""); + sb.append(Utils.isEmpty(n.getName()) ? "(unnamed)" : n.getName()); + if (n.isLoop()) { + sb.append(" [loop]"); + } + if (n.isGroupBy()) { + sb.append(" [group-by]"); + } + if (!Utils.isEmpty(n.getMappedField())) { + sb.append(" ← ").append(n.getMappedField()); + } else if (!Utils.isEmpty(n.getDefaultValue())) { + sb.append(" = ").append(n.getDefaultValue()); + } + return sb.toString(); + } + + private void refreshSelectedTreeItemLabel() { + TreeItem[] sel = wTree.getSelection(); + if (sel.length == 0) { + return; + } + XmlNode n = (XmlNode) sel[0].getData(); + if (n != null) { + sel[0].setText(formatNodeLabel(n)); + } + } + + private XmlNode currentSelection() { + TreeItem[] sel = wTree.getSelection(); + return sel.length == 0 ? null : (XmlNode) sel[0].getData(); + } + + /** Selects the tree item that holds the given node. */ + private void selectNode(XmlNode target) { + TreeItem found = findTreeItem(wTree.getItems(), target); + if (found != null) { + wTree.setSelection(found); + handleSelectionChanged(); + } + } + + private TreeItem findTreeItem(TreeItem[] items, XmlNode target) { + for (TreeItem ti : items) { + if (ti.getData() == target) { + return ti; + } + TreeItem inner = findTreeItem(ti.getItems(), target); + if (inner != null) { + return inner; + } + } + return null; + } + + /** Finds the parent node of {@code target} in the model, or null for root. */ + private XmlNode findParent(XmlNode target) { + return findParentRecursive(rootNode, target); + } + + private XmlNode findParentRecursive(XmlNode current, XmlNode target) { + if (current == null || current.getChildren() == null) { + return null; + } + for (XmlNode c : current.getChildren()) { + if (c == target) { + return current; + } + XmlNode deeper = findParentRecursive(c, target); + if (deeper != null) { + return deeper; + } + } + return null; + } + + // --------------------------------------------------------------------------- + // Misc + // --------------------------------------------------------------------------- + + private void refreshInputFieldsList() { + if (wInputFields == null) { + return; + } + wInputFields.removeAll(); + for (String f : inputFields) { + wInputFields.add(f); + } + } + + private void refreshMappedFieldCombo() { + if (wpMappedField == null) { + return; + } + String current = wpMappedField.getText(); + wpMappedField.removeAll(); + wpMappedField.add(""); + for (String f : inputFields) { + wpMappedField.add(f); + } + wpMappedField.setText(current == null ? "" : current); + } + + private void fireChanged() { + if (changeListener != null) { + changeListener.onChanged(); + } + } + + private static XmlNode defaultRoot() { + XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); + XmlNode loop = new XmlNode("Row", XmlNode.NodeKind.Element); + loop.setLoop(true); + root.addChild(loop); + return root; + } + + /** Returns the variables resolver, currently unused but kept for future preview features. */ + IVariables getVariables() { + return variables; + } +} diff --git a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties index 3f7f5ab6c8..95bebc2ff8 100644 --- a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties +++ b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties @@ -52,9 +52,87 @@ AdvancedXMLOutputMeta.CheckResult.TreeOk=The XML tree is structurally valid. AdvancedXMLOutputMeta.CheckResult.ExpectedInputOk=Transform is connected to a previous transform receiving rows. AdvancedXMLOutputMeta.CheckResult.ExpectedInputError=No input received from previous transforms. -AdvancedXMLOutputDialog.DialogTitle=Advanced XML output +# ------------------------------------------------------------ +# Dialog +# ------------------------------------------------------------ +AdvancedXMLOutputDialog.Shell.Title=Advanced XML output +AdvancedXMLOutputDialog.TransformName.Label=Transform name +AdvancedXMLOutputDialog.ErrorGettingFields=Unable to read fields from the previous transform. +AdvancedXMLOutputDialog.ErrorGettingFields.Title=Field lookup failed + +# Tabs +AdvancedXMLOutputDialog.FileTab.Title=File +AdvancedXMLOutputDialog.ContentTab.Title=Content +AdvancedXMLOutputDialog.TreeTab.Title=XML Tree + +# File tab AdvancedXMLOutputDialog.Filename.Label=Filename AdvancedXMLOutputDialog.Extension.Label=Extension AdvancedXMLOutputDialog.Encoding.Label=Encoding -AdvancedXMLOutputDialog.AddFileToResult.Label=Add filename to result -AdvancedXMLOutputDialog.Notice.Label=Note: the XML tree designer will be added in a follow-up release. For now, configure the tree by editing the transform metadata directly. +AdvancedXMLOutputDialog.AddTransformnr.Label=Include transform copy number in filename +AdvancedXMLOutputDialog.AddDate.Label=Include date in filename +AdvancedXMLOutputDialog.AddTime.Label=Include time in filename +AdvancedXMLOutputDialog.SpecifyFormat.Label=Specify custom date/time format +AdvancedXMLOutputDialog.DateTimeFormat.Label=Date/time format +AdvancedXMLOutputDialog.SplitEvery.Label=Split every N rows +AdvancedXMLOutputDialog.Zipped.Label=Zip output file +AdvancedXMLOutputDialog.DoNotOpenAtInit.Label=Do not open new file at start +AdvancedXMLOutputDialog.DoNotCreateEmptyFile.Label=Do not create file if no rows +AdvancedXMLOutputDialog.AddToResult.Label=Add filename to result +AdvancedXMLOutputDialog.ShowFiles.Button=Show file name(s) ... +AdvancedXMLOutputDialog.ShowFiles.Title=Output file name preview +AdvancedXMLOutputDialog.ShowFiles.Message=The following {0} file name(s) will be produced: +AdvancedXMLOutputDialog.ShowFiles.Empty=No file names match the current settings. + +# Content tab +AdvancedXMLOutputDialog.CompactFile.Label=Compact (no whitespace between elements) +AdvancedXMLOutputDialog.BlankLineAfterDecl.Label=Blank line after XML declaration +AdvancedXMLOutputDialog.CreateEmptyElement.Label=Emit empty elements +AdvancedXMLOutputDialog.CreateAttributeIfNull.Label=Emit attribute when value is null +AdvancedXMLOutputDialog.CreateAttributeIfUnmapped.Label=Emit attribute when no field is mapped +AdvancedXMLOutputDialog.Trim.Label=Trim leading/trailing whitespace +AdvancedXMLOutputDialog.DefaultDecimal.Label=Default decimal separator +AdvancedXMLOutputDialog.DefaultGroup.Label=Default grouping separator +AdvancedXMLOutputDialog.GenerateXsd.Label=Generate sibling XSD file +AdvancedXMLOutputDialog.DoctypeRoot.Label=DOCTYPE root element +AdvancedXMLOutputDialog.DoctypeSystem.Label=DOCTYPE system identifier +AdvancedXMLOutputDialog.DoctypePublic.Label=DOCTYPE public identifier +AdvancedXMLOutputDialog.XslHref.Label=XSL stylesheet href +AdvancedXMLOutputDialog.XslType.Label=XSL stylesheet type + +# Tree tab +AdvancedXMLOutputDialog.InputFields.Label=Input fields +AdvancedXMLOutputDialog.GetFields.Button=Get fields + +AdvancedXMLOutputDialog.Toolbar.AddElement=+ Element +AdvancedXMLOutputDialog.Toolbar.AddElement.Tooltip=Add a child element to the selected node +AdvancedXMLOutputDialog.Toolbar.AddAttribute=+ Attribute +AdvancedXMLOutputDialog.Toolbar.AddAttribute.Tooltip=Add an attribute to the selected element +AdvancedXMLOutputDialog.Toolbar.AddFragment=+ Fragment +AdvancedXMLOutputDialog.Toolbar.AddFragment.Tooltip=Add a child node whose mapped field is parsed as XML +AdvancedXMLOutputDialog.Toolbar.Delete=Delete +AdvancedXMLOutputDialog.Toolbar.Delete.Tooltip=Delete the selected node and its descendants +AdvancedXMLOutputDialog.Toolbar.MoveUp=Up +AdvancedXMLOutputDialog.Toolbar.MoveUp.Tooltip=Move the selected node up among its siblings +AdvancedXMLOutputDialog.Toolbar.MoveDown=Down +AdvancedXMLOutputDialog.Toolbar.MoveDown.Tooltip=Move the selected node down among its siblings +AdvancedXMLOutputDialog.Toolbar.ToggleLoop=Loop +AdvancedXMLOutputDialog.Toolbar.ToggleLoop.Tooltip=Toggle the loop flag (exactly one node in the tree must be the loop) +AdvancedXMLOutputDialog.Toolbar.ToggleGroupBy=Group-by +AdvancedXMLOutputDialog.Toolbar.ToggleGroupBy.Tooltip=Toggle the group-by flag on the selected ancestor of the loop + +# Tree node properties +AdvancedXMLOutputDialog.Properties.Name=Name +AdvancedXMLOutputDialog.Properties.Namespace=Namespace URI +AdvancedXMLOutputDialog.Properties.Kind=Kind +AdvancedXMLOutputDialog.Properties.MappedField=Mapped field +AdvancedXMLOutputDialog.Properties.DefaultValue=Default value +AdvancedXMLOutputDialog.Properties.Format=Format +AdvancedXMLOutputDialog.Properties.Length=Length +AdvancedXMLOutputDialog.Properties.Precision=Precision +AdvancedXMLOutputDialog.Properties.Currency=Currency +AdvancedXMLOutputDialog.Properties.Decimal=Decimal +AdvancedXMLOutputDialog.Properties.Grouping=Grouping +AdvancedXMLOutputDialog.Properties.Loop=Loop +AdvancedXMLOutputDialog.Properties.GroupBy=Group-by +AdvancedXMLOutputDialog.Properties.ForceCreate=Force create diff --git a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl b/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl new file mode 100644 index 0000000000..fa6fbd981e --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl @@ -0,0 +1,260 @@ + + + + + advanced-xml-output-basic + Y + Minimal Advanced XML Output example: emit a flat <customers><customer>... document with a sibling .xsd schema. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 72 + 32 + 16 + Writes a flat <customers><customer>... document together with a sibling +.xsd describing the structure. + +Output: ${java.io.tmpdir}/advanced-xml-output-basic.xml (+ .xsd) + 540 + + + + + customers grid + Advanced XML output + Y + + + + customers grid + DataGrid + Four sample customers with mixed data types. + Y + + 1 + + none + + + + + 1 + Alice + Anderson + 2024-01-15 + true + + + 2 + Bob + Brown + 2024-02-03 + false + + + 3 + Carol + Carter + 2024-04-21 + true + + + 4 + Daniel + Davis + 2024-06-30 + false + + + + + -1 + -1 + + N + id + + + + Integer + + + -1 + -1 + + N + firstName + + + + String + + + -1 + -1 + + N + lastName + + + + String + + + -1 + -1 + + N + signupDate + yyyy-MM-dd + + + Date + + + -1 + -1 + + N + premium + + + + Boolean + + + + + 96 + 176 + + + + Advanced XML output + AdvancedXMLOutput + Pretty-prints the rows as <customer> elements under a <customers> root, alongside a generated XSD. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/advanced-xml-output-basic + xml + 0 + N + N + N + N + Y + N + Y + N + + + + customers + Element + + + customer + Element + Y + + + id + Attribute + id + + + firstName + Element + firstName + + + lastName + Element + lastName + + + signupDate + Element + signupDate + yyyy-MM-dd + + + premium + Element + premium + + + + + + + + 320 + 176 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl b/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl new file mode 100644 index 0000000000..45c930cea4 --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl @@ -0,0 +1,359 @@ + + + + + advanced-xml-output-grouped + Y + Advanced XML Output example with group-by, attributes, a default XML namespace, a DOCTYPE, an XSL stylesheet processing instruction, and a generated XSD. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 120 + 32 + 16 + Group-by example: order rows are sorted by orderId then folded under one +<order> element per id, with their item rows nested inside. + +The transform also emits: + - a default XML namespace on the root, + - a DOCTYPE declaration, + - an <?xml-stylesheet?> processing instruction, + - a sibling .xsd schema next to the .xml output. + +Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) + 620 + + + + + order lines grid + sort by orderId + Y + + + sort by orderId + Advanced XML output + Y + + + + order lines grid + DataGrid + Three orders, six line items. Out of order on purpose so the Sort step does some work. + Y + + 1 + + none + + + + + 1001 + Alice Anderson + SKU-100 + Widget + 2 + 9.99 + + + 1003 + Carol Carter + SKU-300 + Gadget + 1 + 49.50 + + + 1001 + Alice Anderson + SKU-150 + Widget refill pack + 3 + 4.25 + + + 1002 + Bob Brown + SKU-200 + Gizmo + 1 + 19.95 + + + 1003 + Carol Carter + SKU-310 + Gadget battery + 2 + 3.50 + + + 1002 + Bob Brown + SKU-250 + Gizmo case + 1 + 5.00 + + + + + -1 + -1 + + N + orderId + + + + Integer + + + -1 + -1 + + N + customer + + + + String + + + -1 + -1 + + N + productSku + + + + String + + + -1 + -1 + + N + productName + + + + String + + + -1 + -1 + + N + qty + + + + Integer + + + -1 + 2 + + N + unitPrice + 0.00 + + . + Number + + + + + 96 + 208 + + + + sort by orderId + SortRows + Group-by collapses consecutive rows; sort first so all line items of the same order are adjacent. + Y + + 1 + + none + + + + + orderId + Y + N + N + 0 + N + + + ${java.io.tmpdir} + advanced-xml-output-grouped-sort + 1000000 + + N + N + + + + 320 + 208 + + + + Advanced XML output + AdvancedXMLOutput + orders/order/lines/line tree with group-by on orderId and loop on line. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + Y + . + + Y + orders + orders.dtd + + orders.xsl + text/xsl + + ${java.io.tmpdir}/advanced-xml-output-grouped + xml + 0 + N + N + N + N + Y + N + Y + N + + + + orders + http://example.com/orders + Element + + + order + Element + Y + orderId + + + id + Attribute + orderId + + + customer + Element + customer + + + lines + Element + Y + + + line + Element + Y + + + sku + Attribute + productSku + + + name + Element + productName + + + qty + Element + qty + + + price + Element + unitPrice + 0.00 + + + + + + + + + + + + 544 + 208 + + + + + + diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java new file mode 100644 index 0000000000..29f48158b5 --- /dev/null +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Loads each shipped sample pipeline (in {@code src/main/samples/transforms/}) and verifies that + * its {@code AdvancedXMLOutput} transform deserializes to a sensible {@link AdvancedXmlOutputMeta} + * (typed-tree shape, mandatory loop element, recognized properties). + * + *

This is a structural smoke test, not a full pipeline run: it catches typos and schema drift in + * the hand-written sample files before they ever reach the user. + */ +class AdvancedXmlOutputSamplesTest { + + private static final Path SAMPLES_DIR = + Path.of("src", "main", "samples", "transforms").toAbsolutePath(); + + @BeforeAll + static void setup() throws Exception { + HopClientEnvironment.init(); + } + + @Test + void basicSampleParsesAndHasExpectedShape() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("advanced-xml-output-basic.hpl"); + + assertEquals( + "${java.io.tmpdir}/advanced-xml-output-basic", meta.getFileSupport().getFileName()); + assertEquals("xml", meta.getFileSupport().getExtension()); + assertTrue(meta.isGenerateXsd(), "basic sample is meant to demonstrate XSD generation"); + + XmlNode root = meta.getRootNode(); + assertNotNull(root); + assertEquals("customers", root.getName()); + assertEquals(1, root.getChildren().size()); + XmlNode customer = root.getChildren().get(0); + assertEquals("customer", customer.getName()); + assertTrue(customer.isLoop(), "the element must be the row loop"); + // 1 attribute (id) + 4 element fields + assertEquals(5, customer.getChildren().size()); + assertEquals(XmlNode.NodeKind.Attribute, customer.getChildren().get(0).getKind()); + assertEquals("id", customer.getChildren().get(0).getName()); + } + + @Test + void groupedSampleParsesAndHasExpectedShape() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("advanced-xml-output-grouped.hpl"); + + assertTrue(meta.isGenerateXsd()); + assertEquals("orders", meta.getDoctypeRootElement()); + assertEquals("orders.dtd", meta.getDoctypeSystemId()); + assertEquals("orders.xsl", meta.getXslStylesheetHref()); + + XmlNode root = meta.getRootNode(); + assertNotNull(root); + assertEquals("orders", root.getName()); + assertEquals("http://example.com/orders", root.getNamespace()); + + XmlNode order = root.getChildren().get(0); + assertEquals("order", order.getName()); + assertTrue(order.isGroupBy(), " must be flagged group-by"); + assertEquals("orderId", order.getMappedField()); + + // order > [id @, customer, lines] + assertEquals(3, order.getChildren().size()); + XmlNode lines = order.getChildren().get(2); + assertEquals("lines", lines.getName()); + assertTrue(lines.isForceCreate(), " wraps the loop and is marked force-create"); + + XmlNode line = lines.getChildren().get(0); + assertEquals("line", line.getName()); + assertTrue(line.isLoop(), " must be the row loop"); + + // Validator clean run: assert exactly one loop, lines/line/sku/name/qty/price etc. + List structural = AdvancedXmlOutputValidator.validate(root, null); + // With no input row meta we expect no field-existence errors, only structural checks pass. + assertTrue( + structural.isEmpty(), + "structural validation should be clean for the grouped sample, got: " + structural); + } + + @Test + void allSamplesAreWellFormedXml() throws Exception { + List samples = listSamples(); + assertTrue(samples.size() >= 2, "expected at least two sample pipelines"); + for (Path p : samples) { + Document doc = XmlHandler.loadXmlString(Files.readString(p)); + assertNotNull(doc, "could not parse " + p.getFileName()); + assertEquals("pipeline", doc.getDocumentElement().getNodeName(), "wrong root in " + p); + } + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private static AdvancedXmlOutputMeta loadAdvancedXmlOutput(String pipelineFilename) + throws Exception { + Path path = SAMPLES_DIR.resolve(pipelineFilename); + Document doc = XmlHandler.loadXmlString(Files.readString(path)); + Node transformNode = findAdvancedXmlOutputTransformNode(doc); + assertNotNull(transformNode, "no AdvancedXMLOutput transform found in " + pipelineFilename); + + return XmlMetadataUtil.deSerializeFromXml( + transformNode, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); + } + + private static Node findAdvancedXmlOutputTransformNode(Document doc) { + NodeList transforms = doc.getElementsByTagName("transform"); + for (int i = 0; i < transforms.getLength(); i++) { + Node t = transforms.item(i); + String type = XmlHandler.getTagValue(t, "type"); + if ("AdvancedXMLOutput".equals(type)) { + return t; + } + } + return null; + } + + private static List listSamples() throws Exception { + List out = new ArrayList<>(); + try (var stream = Files.list(SAMPLES_DIR)) { + stream + .filter(p -> p.getFileName().toString().endsWith(".hpl")) + .filter(p -> p.getFileName().toString().startsWith("advanced-xml-output-")) + .sorted() + .forEach(out::add); + } + return out; + } +} diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java index d90903845a..c42563eea6 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java @@ -25,6 +25,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.RowMetaAndData; import org.apache.hop.core.row.IRowMeta; @@ -142,6 +144,153 @@ void testCompactFileHasNoNewlinesBetweenElements(@TempDir Path tempDir) throws E assertFalse(body.contains("\n"), "Compact mode should not contain newlines: " + body); } + // --------------------------------------------------------------------------- + // Namespace inheritance: only the root declares xmlns; children inherit it. + // --------------------------------------------------------------------------- + + @Test + void testRootDefaultNamespaceIsInheritedByChildren(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("ns"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.getRootNode().setNamespace("http://example.com/customers"); + + runPipeline(meta, buildFlatRows("Alice", 30)); + + String xml = readWrittenFile(output); + // Root declares the default namespace exactly once + assertTrue( + xml.contains("xmlns=\"http://example.com/customers\""), + "expected root xmlns declaration, got: " + xml); + assertEquals( + 1, + count(xml, "xmlns=\"http://example.com/customers\""), + "the namespace should only be declared on the root: " + xml); + // Child elements still appear (the writer didn't fail on an "unbound" URI) + assertTrue(xml.contains(""), "row element missing: " + xml); + assertTrue(xml.contains("Alice")); + } + + // --------------------------------------------------------------------------- + // DOCTYPE + XSL stylesheet PI + // --------------------------------------------------------------------------- + + @Test + void testDoctypeAndXslPiAreEmitted(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("doc"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.setDoctypeRootElement("Rows"); + meta.setDoctypeSystemId("rows.dtd"); + meta.setXslStylesheetHref("rows.xsl"); + + runPipeline(meta, buildFlatRows("a", 1)); + String xml = readWrittenFile(output); + assertTrue(xml.contains(""), + "xml-stylesheet PI missing: " + xml); + } + + // --------------------------------------------------------------------------- + // Force create / create-attribute-if-null / create-empty-element + // --------------------------------------------------------------------------- + + @Test + void testForceCreateEmitsStaticElementWithDefaultValue(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("force"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + XmlNode loop = meta.getRootNode().getChildren().get(0); + XmlNode note = new XmlNode("note", XmlNode.NodeKind.Element); + note.setForceCreate(true); + note.setDefaultValue("(none)"); + loop.addChild(note); + + runPipeline(meta, buildFlatRows("Alice", 30)); + + String xml = readWrittenFile(output); + assertTrue(xml.contains("(none)"), "expected force-created element: " + xml); + } + + // --------------------------------------------------------------------------- + // Split-every produces multiple files + // --------------------------------------------------------------------------- + + @Test + void testSplitEveryProducesMultipleFiles(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("split"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.getFileSupport().setSplitEvery(2); + + runPipeline(meta, buildFlatRows("A", 1, "B", 2, "C", 3, "D", 4, "E", 5)); + + // Expect 3 files: 2 rows, 2 rows, 1 row + Path f1 = Path.of(output + "_00001.xml"); + Path f2 = Path.of(output + "_00002.xml"); + Path f3 = Path.of(output + "_00003.xml"); + assertTrue(Files.exists(f1), "first split file missing: " + f1); + assertTrue(Files.exists(f2), "second split file missing: " + f2); + assertTrue(Files.exists(f3), "third split file missing: " + f3); + assertEquals(2, count(Files.readString(f1), "")); + assertEquals(2, count(Files.readString(f2), "")); + assertEquals(1, count(Files.readString(f3), "")); + } + + // --------------------------------------------------------------------------- + // Zipped output + // --------------------------------------------------------------------------- + + @Test + void testZippedOutputContainsValidXml(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("zipped"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.getFileSupport().setZipped(true); + + runPipeline(meta, buildFlatRows("a", 1, "b", 2)); + + Path zip = Path.of(output + ".zip"); + assertTrue(Files.exists(zip), "zip archive missing: " + zip); + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(zip))) { + ZipEntry e = zis.getNextEntry(); + assertTrue(e != null && e.getName().endsWith(".xml"), "first entry is not .xml: " + e); + String content = new String(zis.readAllBytes()); + assertTrue(content.contains("")); + assertEquals(2, count(content, "")); + } + } + + // --------------------------------------------------------------------------- + // XSD generation + // --------------------------------------------------------------------------- + + @Test + void testGenerateXsdProducesSiblingSchema(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("schema"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.setGenerateXsd(true); + + runPipeline(meta, buildFlatRows("a", 1, "b", 2)); + + Path xsd = Path.of(output + ".xsd"); + assertTrue(Files.exists(xsd), "Sibling XSD should have been written: " + xsd); + String content = Files.readString(xsd); + assertTrue(content.contains("()); + + Path xsd = Path.of(output + ".xsd"); + assertFalse(Files.exists(xsd), "Empty input must not produce an XSD: " + xsd); + } + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriterTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriterTest.java new file mode 100644 index 0000000000..e025848677 --- /dev/null +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputXsdWriterTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Unit tests for {@link AdvancedXmlOutputXsdWriter}: derives XSD type for various Hop value types + * and renders a minimal but well-formed schema document for a representative tree. + */ +class AdvancedXmlOutputXsdWriterTest { + + @BeforeAll + static void setup() throws Exception { + HopEnvironment.init(); + } + + @Test + void testTypeMapping() { + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("s")); + rm.addValueMeta(new ValueMetaInteger("i")); + rm.addValueMeta(new ValueMetaNumber("n")); + rm.addValueMeta(new ValueMetaDate("d")); + rm.addValueMeta(new ValueMetaBoolean("b")); + + assertEquals("string", typeFor("s", rm)); + assertEquals("long", typeFor("i", rm)); + assertEquals("decimal", typeFor("n", rm)); + assertEquals("dateTime", typeFor("d", rm)); + assertEquals("boolean", typeFor("b", rm)); + assertEquals("string", typeFor("missing", rm)); + assertEquals("string", typeFor(null, rm)); + } + + private static String typeFor(String mappedField, IRowMeta rm) { + XmlNode n = new XmlNode("x", XmlNode.NodeKind.Element); + if (mappedField != null) { + n.setMappedField(mappedField); + } + return AdvancedXmlOutputXsdWriter.xsdSimpleTypeFor(n, rm); + } + + @Test + void testGeneratesSchemaForFlatTree(@TempDir Path tempDir) throws Exception { + XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); + XmlNode row = new XmlNode("Row", XmlNode.NodeKind.Element); + row.setLoop(true); + XmlNode name = new XmlNode("name", XmlNode.NodeKind.Element); + name.setMappedField("name"); + XmlNode age = new XmlNode("age", XmlNode.NodeKind.Element); + age.setMappedField("age"); + XmlNode active = new XmlNode("active", XmlNode.NodeKind.Attribute); + active.setMappedField("active"); + row.addChild(active); + row.addChild(name); + row.addChild(age); + root.addChild(row); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("name")); + rm.addValueMeta(new ValueMetaInteger("age")); + rm.addValueMeta(new ValueMetaBoolean("active")); + + Path xsdPath = tempDir.resolve("flat.xsd"); + AdvancedXmlOutputXsdWriter.write(xsdPath.toString(), new Variables(), "UTF-8", root, rm); + String xsd = Files.readString(xsdPath); + + assertTrue(xsd.contains(" unbounded + assertTrue(xsd.contains("maxOccurs=\"unbounded\""), "loop should be unbounded"); + // Type mappings on simple elements + assertTrue(xsd.contains("type=\"xs:string\""), "expected xs:string for name"); + assertTrue(xsd.contains("type=\"xs:long\""), "expected xs:long for age"); + // Attribute on Row + assertTrue(xsd.contains("', rowsIdx)); + assertFalse(rowsHeader.contains("maxOccurs"), "root element should not have maxOccurs"); + assertTrue(rowIdx > rowsIdx); + } + + @Test + void testGeneratesSchemaWithTargetNamespace(@TempDir Path tempDir) throws Exception { + XmlNode root = new XmlNode("Catalog", XmlNode.NodeKind.Element); + root.setNamespace("http://example.com/catalog"); + XmlNode item = new XmlNode("Item", XmlNode.NodeKind.Element); + item.setLoop(true); + XmlNode title = new XmlNode("Title", XmlNode.NodeKind.Element); + title.setMappedField("title"); + item.addChild(title); + root.addChild(item); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("title")); + + Path xsdPath = tempDir.resolve("ns.xsd"); + AdvancedXmlOutputXsdWriter.write(xsdPath.toString(), new Variables(), "UTF-8", root, rm); + String xsd = Files.readString(xsdPath); + + assertTrue( + xsd.contains("targetNamespace=\"http://example.com/catalog\""), "targetNamespace missing"); + assertTrue(xsd.contains("elementFormDefault=\"qualified\""), "qualified default missing"); + } + + @Test + void testGroupByElementIsUnbounded(@TempDir Path tempDir) throws Exception { + XmlNode orders = new XmlNode("orders", XmlNode.NodeKind.Element); + XmlNode order = new XmlNode("order", XmlNode.NodeKind.Element); + order.setGroupBy(true); + order.setMappedField("orderId"); + XmlNode item = new XmlNode("item", XmlNode.NodeKind.Element); + item.setLoop(true); + XmlNode name = new XmlNode("name", XmlNode.NodeKind.Element); + name.setMappedField("itemName"); + item.addChild(name); + order.addChild(item); + orders.addChild(order); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaInteger("orderId")); + rm.addValueMeta(new ValueMetaString("itemName")); + + Path xsdPath = tempDir.resolve("orders.xsd"); + AdvancedXmlOutputXsdWriter.write(xsdPath.toString(), new Variables(), "UTF-8", orders, rm); + String xsd = Files.readString(xsdPath); + + // Both order (group-by) and item (loop) should be unbounded + int unbounded = 0; + int idx = 0; + while ((idx = xsd.indexOf("maxOccurs=\"unbounded\"", idx)) >= 0) { + unbounded++; + idx++; + } + assertTrue(unbounded >= 2, "expected at least two unbounded children, got " + unbounded); + } +} From 99c3fbbfe2bf986cec215019f10703b86140f70c Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Fri, 8 May 2026 16:54:22 +0200 Subject: [PATCH 3/9] fixes, integration tests, additional samples --- docs/hop-user-manual/modules/ROOT/nav.adoc | 1 + ...dxmloutput.adoc => xmloutputadvanced.adoc} | 8 +- .../xml/0011-xml-output-advanced-basic.hpl | 237 +++++++++++ .../xml/0012-xml-output-advanced-grouped.hpl | 330 +++++++++++++++ ...013-xml-output-advanced-multi-group-by.hpl | 359 ++++++++++++++++ ...-xml-output-advanced-document-fragment.hpl | 211 ++++++++++ .../xml/0015-xml-output-advanced-split.hpl | 172 ++++++++ .../xml/0016-xml-output-advanced-compact.hpl | 241 +++++++++++ .../xml/0017-xml-output-advanced-zipped.hpl | 185 +++++++++ .../xml/files/expected/0011-basic.xml | 7 + .../xml/files/expected/0011-basic.xml.bak | 7 + .../xml/files/expected/0011-basic.xsd | 2 + .../xml/files/expected/0012-grouped.xml | 23 ++ .../xml/files/expected/0012-grouped.xsd | 2 + .../files/expected/0013-multi-group-by.xml | 31 ++ .../files/expected/0013-multi-group-by.xsd | 2 + .../files/expected/0014-document-fragment.xml | 6 + .../files/expected/0014-document-fragment.xsd | 2 + .../xml/files/expected/0015-split_0_00001.xml | 8 + .../xml/files/expected/0015-split_0_00002.xml | 8 + .../xml/files/expected/0015-split_0_00003.xml | 5 + .../xml/files/expected/0016-compact.xml | 1 + .../xml/files/expected/0017-zipped.xml | 7 + .../xml/files/expected/0017-zipped.xsd | 2 + .../main-0011-xml-output-advanced-basic.hwf | 210 ++++++++++ .../main-0012-xml-output-advanced-grouped.hwf | 210 ++++++++++ ...013-xml-output-advanced-multi-group-by.hwf | 210 ++++++++++ ...-xml-output-advanced-document-fragment.hwf | 210 ++++++++++ .../main-0015-xml-output-advanced-split.hwf | 248 +++++++++++ .../main-0016-xml-output-advanced-compact.hwf | 172 ++++++++ .../main-0017-xml-output-advanced-zipped.hwf | 262 ++++++++++++ integration-tests/xml/output/.gitignore | 3 + .../advancedxmloutput/AdvancedXmlOutput.java | 73 ++-- .../AdvancedXmlOutputDialog.java | 2 +- .../AdvancedXmlOutputMeta.java | 4 +- .../xml/advancedxmloutput/XmlNode.java | 2 +- .../messages/messages_en_US.properties | 4 +- ...asic.hpl => xml-output-advanced-basic.hpl} | 12 +- .../xml-output-advanced-compact.hpl | 270 ++++++++++++ .../xml-output-advanced-document-fragment.hpl | 236 +++++++++++ ...ed.hpl => xml-output-advanced-grouped.hpl} | 14 +- .../xml-output-advanced-multi-group-by.hpl | 385 ++++++++++++++++++ .../transforms/xml-output-advanced-split.hpl | 197 +++++++++ .../transforms/xml-output-advanced-zipped.hpl | 211 ++++++++++ .../AdvancedXmlOutputSamplesTest.java | 143 ++++++- .../AdvancedXmlOutputTest.java | 58 ++- 46 files changed, 4925 insertions(+), 68 deletions(-) rename docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/{advancedxmloutput.adoc => xmloutputadvanced.adoc} (93%) create mode 100644 integration-tests/xml/0011-xml-output-advanced-basic.hpl create mode 100644 integration-tests/xml/0012-xml-output-advanced-grouped.hpl create mode 100644 integration-tests/xml/0013-xml-output-advanced-multi-group-by.hpl create mode 100644 integration-tests/xml/0014-xml-output-advanced-document-fragment.hpl create mode 100644 integration-tests/xml/0015-xml-output-advanced-split.hpl create mode 100644 integration-tests/xml/0016-xml-output-advanced-compact.hpl create mode 100644 integration-tests/xml/0017-xml-output-advanced-zipped.hpl create mode 100644 integration-tests/xml/files/expected/0011-basic.xml create mode 100644 integration-tests/xml/files/expected/0011-basic.xml.bak create mode 100644 integration-tests/xml/files/expected/0011-basic.xsd create mode 100644 integration-tests/xml/files/expected/0012-grouped.xml create mode 100644 integration-tests/xml/files/expected/0012-grouped.xsd create mode 100644 integration-tests/xml/files/expected/0013-multi-group-by.xml create mode 100644 integration-tests/xml/files/expected/0013-multi-group-by.xsd create mode 100644 integration-tests/xml/files/expected/0014-document-fragment.xml create mode 100644 integration-tests/xml/files/expected/0014-document-fragment.xsd create mode 100644 integration-tests/xml/files/expected/0015-split_0_00001.xml create mode 100644 integration-tests/xml/files/expected/0015-split_0_00002.xml create mode 100644 integration-tests/xml/files/expected/0015-split_0_00003.xml create mode 100644 integration-tests/xml/files/expected/0016-compact.xml create mode 100644 integration-tests/xml/files/expected/0017-zipped.xml create mode 100644 integration-tests/xml/files/expected/0017-zipped.xsd create mode 100644 integration-tests/xml/main-0011-xml-output-advanced-basic.hwf create mode 100644 integration-tests/xml/main-0012-xml-output-advanced-grouped.hwf create mode 100644 integration-tests/xml/main-0013-xml-output-advanced-multi-group-by.hwf create mode 100644 integration-tests/xml/main-0014-xml-output-advanced-document-fragment.hwf create mode 100644 integration-tests/xml/main-0015-xml-output-advanced-split.hwf create mode 100644 integration-tests/xml/main-0016-xml-output-advanced-compact.hwf create mode 100644 integration-tests/xml/main-0017-xml-output-advanced-zipped.hwf create mode 100644 integration-tests/xml/output/.gitignore rename plugins/transforms/xml/src/main/samples/transforms/{advanced-xml-output-basic.hpl => xml-output-advanced-basic.hpl} (94%) create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-compact.hpl create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-document-fragment.hpl rename plugins/transforms/xml/src/main/samples/transforms/{advanced-xml-output-grouped.hpl => xml-output-advanced-grouped.hpl} (95%) create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-multi-group-by.hpl create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-split.hpl create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-zipped.hpl diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 40fedf4405..d87d5b3289 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -284,6 +284,7 @@ under the License. *** xref:pipeline/transforms/xmlinputstream.adoc[XML Input Stream (StAX)] *** xref:pipeline/transforms/xmljoin.adoc[XML Join] *** xref:pipeline/transforms/xmloutput.adoc[XML Output] +*** xref:pipeline/transforms/xmloutputadvanced.adoc[XML Output (Advanced)] *** xref:pipeline/transforms/xsdvalidator.adoc[XSD Validator] *** xref:pipeline/transforms/xslt.adoc[XSL Transformation] *** xref:pipeline/transforms/yamlinput.adoc[Yaml Input] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc similarity index 93% rename from docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc rename to docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc index 5a0e4669be..946b46b45b 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/advancedxmloutput.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc @@ -16,20 +16,20 @@ under the License. //// :documentationPath: /pipeline/transforms/ :language: en_US -:description: The Advanced XML Output transform writes rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. +:description: The XML Output (Advanced) transform writes rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. -= image:transforms/icons/AXO.svg[Advanced XML Output transform Icon, role="image-doc-icon"] Advanced XML Output += image:transforms/icons/AXO.svg[XML Output (Advanced) transform Icon, role="image-doc-icon"] XML Output (Advanced) [%noheader,cols="3a,1a", role="table-no-borders" ] |=== | == Description -The Advanced XML Output transform writes rows from any source to one or more XML files, using a hierarchical, user-defined XML tree. +The XML Output (Advanced) transform writes rows from any source to one or more XML files, using a hierarchical, user-defined XML tree. The XML tree is a recursive structure of elements, attributes and document-fragment nodes. Exactly one element in the tree must be marked as the row-*loop*: each input row produces one occurrence of that element with its full subtree. Optionally, ancestors of the loop can be marked as *group-by*: consecutive input rows that share the same group key are emitted under a single occurrence of the group element. -This transform complements the simpler `XML Output` transform. Use `XML Output` for a flat document of repeating rows; use `Advanced XML Output` when you need a deeper, custom-shaped XML structure (loops nested inside groups, attributes at any level, document fragments, namespaces, schema generation). +This transform complements the simpler `XML Output` transform. Use `XML Output` for a flat document of repeating rows; use `XML Output (Advanced)` when you need a deeper, custom-shaped XML structure (loops nested inside groups, attributes at any level, document fragments, namespaces, schema generation). | == Supported Engines diff --git a/integration-tests/xml/0011-xml-output-advanced-basic.hpl b/integration-tests/xml/0011-xml-output-advanced-basic.hpl new file mode 100644 index 0000000000..5c274822c5 --- /dev/null +++ b/integration-tests/xml/0011-xml-output-advanced-basic.hpl @@ -0,0 +1,237 @@ + + + + + 0011-xml-output-advanced-basic + Y + Integration test: minimal XML Output (Advanced) example. Writes a flat <customers><customer>... document with a sibling .xsd schema. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + + + customers grid + XML Output (Advanced) + Y + + + + customers grid + DataGrid + Four sample customers with mixed data types. + Y + + 1 + + none + + + + + 1 + Alice + Anderson + 2024-01-15 + true + + + 2 + Bob + Brown + 2024-02-03 + false + + + 3 + Carol + Carter + 2024-04-21 + true + + + 4 + Daniel + Davis + 2024-06-30 + false + + + + + -1 + -1 + + N + id + + + + Integer + + + -1 + -1 + + N + firstName + + + + String + + + -1 + -1 + + N + lastName + + + + String + + + -1 + -1 + + N + signupDate + yyyy-MM-dd + + + Date + + + -1 + -1 + + N + premium + + + + Boolean + + + + + 96 + 176 + + + + XML Output (Advanced) + AdvancedXMLOutput + Pretty-prints rows as <customer> elements under a <customers> root with a sibling XSD. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0011-basic + xml + 0 + N + N + N + N + Y + N + Y + N + + + + customers + Element + + + customer + Element + Y + + + id + Attribute + id + + + firstName + Element + firstName + + + lastName + Element + lastName + + + signupDate + Element + signupDate + yyyy-MM-dd + + + premium + Element + premium + + + + + + + + 320 + 176 + + + + + + diff --git a/integration-tests/xml/0012-xml-output-advanced-grouped.hpl b/integration-tests/xml/0012-xml-output-advanced-grouped.hpl new file mode 100644 index 0000000000..c4fe05d2ca --- /dev/null +++ b/integration-tests/xml/0012-xml-output-advanced-grouped.hpl @@ -0,0 +1,330 @@ + + + + + 0012-xml-output-advanced-grouped + Y + Integration test: XML Output (Advanced) with group-by, attributes, default namespace, DOCTYPE, XSL PI and a generated XSD. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + + + order lines grid + sort by orderId + Y + + + sort by orderId + XML Output (Advanced) + Y + + + + order lines grid + DataGrid + Three orders, six line items. Out of order on purpose so the Sort step does some work. + Y + + 1 + + none + + + + + 1001 + Alice Anderson + SKU-100 + Widget + 2 + 9.99 + + + 1003 + Carol Carter + SKU-300 + Gadget + 1 + 49.50 + + + 1001 + Alice Anderson + SKU-150 + Widget refill pack + 3 + 4.25 + + + 1002 + Bob Brown + SKU-200 + Gizmo + 1 + 19.95 + + + 1003 + Carol Carter + SKU-310 + Gadget battery + 2 + 3.50 + + + 1002 + Bob Brown + SKU-250 + Gizmo case + 1 + 5.00 + + + + + -1 + -1 + + N + orderId + + + + Integer + + + -1 + -1 + + N + customer + + + + String + + + -1 + -1 + + N + productSku + + + + String + + + -1 + -1 + + N + productName + + + + String + + + -1 + -1 + + N + qty + + + + Integer + + + -1 + 2 + + N + unitPrice + 0.00 + + . + Number + + + + + 96 + 208 + + + + sort by orderId + SortRows + Group-by collapses consecutive rows; sort first so all line items of the same order are adjacent. + Y + + 1 + + none + + + + + orderId + Y + N + N + 0 + N + + + ${java.io.tmpdir} + 0012-xml-output-advanced-grouped-sort + 1000000 + + N + N + + + + 320 + 208 + + + + XML Output (Advanced) + AdvancedXMLOutput + orders/order/lines/line tree with group-by on orderId and loop on line. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + Y + . + + Y + orders + orders.dtd + + orders.xsl + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0012-grouped + xml + 0 + N + N + N + N + Y + N + Y + N + + + + orders + http://example.com/orders + Element + + + order + Element + Y + orderId + + + id + Attribute + orderId + + + customer + Element + customer + + + lines + Element + Y + + + line + Element + Y + + + sku + Attribute + productSku + + + name + Element + productName + + + qty + Element + qty + + + price + Element + unitPrice + 0.00 + + + + + + + + + + + + 544 + 208 + + + + + + diff --git a/integration-tests/xml/0013-xml-output-advanced-multi-group-by.hpl b/integration-tests/xml/0013-xml-output-advanced-multi-group-by.hpl new file mode 100644 index 0000000000..dc5fa5928f --- /dev/null +++ b/integration-tests/xml/0013-xml-output-advanced-multi-group-by.hpl @@ -0,0 +1,359 @@ + + + + + 0013-xml-output-advanced-multi-group-by + Y + Integration test: XML Output (Advanced) with two levels of group-by ancestors above the row loop. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + + order lines grid + sort by region, orderId + Y + + + sort by region, orderId + XML Output (Advanced) + Y + + + + order lines grid + DataGrid + Two regions, four orders, eight line items. + Y + + 1 + + none + + + + + EU + 1001 + Alice Anderson + SKU-100 + 2 + 9.99 + + + NA + 2001 + Diane Davis + SKU-510 + 1 + 14.00 + + + EU + 1001 + Alice Anderson + SKU-150 + 3 + 4.25 + + + EU + 1002 + Bob Brown + SKU-200 + 1 + 19.95 + + + NA + 2002 + Eric Evans + SKU-600 + 4 + 2.50 + + + EU + 1002 + Bob Brown + SKU-250 + 1 + 5.00 + + + NA + 2002 + Eric Evans + SKU-610 + 2 + 3.10 + + + NA + 2001 + Diane Davis + SKU-520 + 2 + 6.50 + + + + + -1 + -1 + + N + region + + + + String + + + -1 + -1 + + N + orderId + + + + Integer + + + -1 + -1 + + N + customer + + + + String + + + -1 + -1 + + N + productSku + + + + String + + + -1 + -1 + + N + qty + + + + Integer + + + -1 + 2 + + N + unitPrice + 0.00 + + . + Number + + + + + 96 + 208 + + + + sort by region, orderId + SortRows + Group-by needs sorted input. Sort by every group key from outermost to innermost. + Y + + 1 + + none + + + + + region + Y + N + N + 0 + N + + + orderId + Y + N + N + 0 + N + + + ${java.io.tmpdir} + 0013-xml-output-advanced-multi-group-by-sort + 1000000 + + N + N + + + + 320 + 208 + + + + XML Output (Advanced) + AdvancedXMLOutput + regions/region(group)/order(group)/lines/line(loop) with attribute @code on the region. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + Y + Y + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0013-multi-group-by + xml + 0 + N + N + N + N + Y + N + Y + N + + + + regions + Element + + + region + Element + Y + region + + + code + Attribute + region + + + order + Element + Y + orderId + + + id + Attribute + orderId + + + customer + Element + customer + + + lines + Element + Y + + + line + Element + Y + + + sku + Attribute + productSku + + + qty + Element + qty + + + price + Element + unitPrice + 0.00 + + + + + + + + + + + + + + 544 + 208 + + + + + + diff --git a/integration-tests/xml/0014-xml-output-advanced-document-fragment.hpl b/integration-tests/xml/0014-xml-output-advanced-document-fragment.hpl new file mode 100644 index 0000000000..8b8294ac2f --- /dev/null +++ b/integration-tests/xml/0014-xml-output-advanced-document-fragment.hpl @@ -0,0 +1,211 @@ + + + + + 0014-xml-output-advanced-document-fragment + Y + Integration test: XML Output (Advanced) DocumentFragment node parses an XML fragment field and inserts it as real XML. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + + products grid + XML Output (Advanced) + Y + + + + products grid + DataGrid + Three products, each with its own pre-built XML fragment of metadata. + Y + + 1 + + none + + + + + SKU-100 + Widget + 9.99 + <weight unit="g">120</weight><color>blue</color><tag>hardware</tag> + + + SKU-200 + Gizmo + 19.95 + <weight unit="g">240</weight><color>red</color><dimensions w="10" h="5" d="3"/> + + + SKU-300 + Gadget + 49.50 + <weight unit="g">560</weight><color>graphite</color><notes priority="low">Premium edition. Special characters like &amp; or &lt; are preserved as text.</notes> + + + + + -1 + -1 + + N + sku + + + + String + + + -1 + -1 + + N + name + + + + String + + + -1 + 2 + + N + price + 0.00 + + . + Number + + + -1 + -1 + + N + extras + + + + String + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + products/product(loop) with an Element name and a DocumentFragment that injects the "extras" field as XML. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0014-document-fragment + xml + 0 + N + N + N + N + Y + N + Y + N + + + + products + Element + + + product + Element + Y + + + sku + Attribute + sku + + + name + Element + name + + + price + Element + price + 0.00 + + + extras + DocumentFragment + extras + + + + + + + + 320 + 192 + + + + + + diff --git a/integration-tests/xml/0015-xml-output-advanced-split.hpl b/integration-tests/xml/0015-xml-output-advanced-split.hpl new file mode 100644 index 0000000000..bb114ae38d --- /dev/null +++ b/integration-tests/xml/0015-xml-output-advanced-split.hpl @@ -0,0 +1,172 @@ + + + + + 0015-xml-output-advanced-split + Y + Integration test: XML Output (Advanced) split-every rolls over to a new output file every N rows. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + + events grid + XML Output (Advanced) + Y + + + + events grid + DataGrid + Twelve numbered events. + Y + + 1 + + none + + + + 1start + 2checkpoint + 3checkpoint + 4checkpoint + 5checkpoint + 6checkpoint + 7checkpoint + 8checkpoint + 9checkpoint + 10checkpoint + 11checkpoint + 12end + + + + -1 + -1 + + N + eventId + + + + Integer + + + -1 + -1 + + N + kind + + + + String + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Split every 5 rows; include the transform copy number in the filename. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0015-split + xml + 5 + Y + N + N + N + Y + N + Y + N + + + + events + Element + + + event + Element + Y + + + id + Attribute + eventId + + + kind + Element + kind + + + + + + + + 320 + 192 + + + + + + diff --git a/integration-tests/xml/0016-xml-output-advanced-compact.hpl b/integration-tests/xml/0016-xml-output-advanced-compact.hpl new file mode 100644 index 0000000000..b9fa604b00 --- /dev/null +++ b/integration-tests/xml/0016-xml-output-advanced-compact.hpl @@ -0,0 +1,241 @@ + + + + + 0016-xml-output-advanced-compact + Y + Integration test: XML Output (Advanced) compact mode + force-create + create-attribute-if-null/unmapped + default values for null fields. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + + products grid + XML Output (Advanced) + Y + + + + products grid + DataGrid + Three products. Some rows leave optional columns NULL to exercise force-create + default values. + Y + + 1 + + none + + + + + SKU-100 + Widget + 9.99 + The blue widget + false + + + SKU-150 + Refill + + + + + + SKU-200 + Gizmo + 19.95 + + true + + + + + -1 + -1 + + N + sku + + + + String + + + -1 + -1 + + N + name + + + + String + + + -1 + 2 + + N + price + 0.00 + + . + Number + + + -1 + -1 + + N + note + + + + String + + + -1 + -1 + + N + discontinued + + + + Boolean + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Compact + create-attr-if-null + create-attr-if-unmapped + force-create with default values. + Y + + 1 + + none + + + UTF-8 + Y + N + Y + Y + Y + Y + N + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0016-compact + xml + 0 + N + N + N + N + Y + N + Y + N + + + + catalog + Element + + + product + Element + Y + + + sku + Attribute + sku + + + currency + Attribute + USD + + + discontinued + Attribute + discontinued + false + Y + + + name + Element + name + + + price + Element + price + 0.00 + 0.00 + Y + + + note + Element + note + (no description) + Y + + + + + + + + 320 + 192 + + + + + + diff --git a/integration-tests/xml/0017-xml-output-advanced-zipped.hpl b/integration-tests/xml/0017-xml-output-advanced-zipped.hpl new file mode 100644 index 0000000000..9f72ae9053 --- /dev/null +++ b/integration-tests/xml/0017-xml-output-advanced-zipped.hpl @@ -0,0 +1,185 @@ + + + + + 0017-xml-output-advanced-zipped + Y + Integration test: XML Output (Advanced) zipped output. The XML lives inside a .zip archive and the sibling .xsd is written next to the .zip. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + + transactions grid + XML Output (Advanced) + Y + + + + transactions grid + DataGrid + A handful of monetary transactions with a EUR currency symbol and grouped thousands. + Y + + 1 + + none + + + + TX-10012026-01-151250.00 + TX-10022026-01-1587500.50 + TX-10032026-01-16250.75 + TX-10042026-01-163450.99 + + + + -1 + -1 + + N + txId + + + + String + + + -1 + -1 + + N + txDate + yyyy-MM-dd + + + Date + + + -1 + 2 + + N + amount + 0.00 + + . + Number + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Per-node currency, decimal and grouping symbols on the <amount> element. Output wrapped in a zip archive, sibling XSD is written next to the .zip. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0017-zipped + xml + 0 + N + N + N + Y + Y + N + Y + N + + + + transactions + Element + + + transaction + Element + Y + + + id + Attribute + txId + + + date + Element + txDate + yyyy-MM-dd + + + amount + Element + amount + #,##0.00 ¤ + EUR + . + , + + + + + + + + 320 + 192 + + + + + + diff --git a/integration-tests/xml/files/expected/0011-basic.xml b/integration-tests/xml/files/expected/0011-basic.xml new file mode 100644 index 0000000000..1ceb007609 --- /dev/null +++ b/integration-tests/xml/files/expected/0011-basic.xml @@ -0,0 +1,7 @@ + + +AliceAnderson2024-01-15Y +BobBrown2024-02-03N +CarolCarter2024-04-21Y +DanielDavis2024-06-30N + diff --git a/integration-tests/xml/files/expected/0011-basic.xml.bak b/integration-tests/xml/files/expected/0011-basic.xml.bak new file mode 100644 index 0000000000..1ceb007609 --- /dev/null +++ b/integration-tests/xml/files/expected/0011-basic.xml.bak @@ -0,0 +1,7 @@ + + +AliceAnderson2024-01-15Y +BobBrown2024-02-03N +CarolCarter2024-04-21Y +DanielDavis2024-06-30N + diff --git a/integration-tests/xml/files/expected/0011-basic.xsd b/integration-tests/xml/files/expected/0011-basic.xsd new file mode 100644 index 0000000000..d1e74e1e21 --- /dev/null +++ b/integration-tests/xml/files/expected/0011-basic.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/files/expected/0012-grouped.xml b/integration-tests/xml/files/expected/0012-grouped.xml new file mode 100644 index 0000000000..e1278c7d8d --- /dev/null +++ b/integration-tests/xml/files/expected/0012-grouped.xml @@ -0,0 +1,23 @@ + + + + +Alice Anderson + +Widget29.99 +Widget refill pack34.25 + + +Bob Brown + +Gizmo119.95 +Gizmo case15.00 + + +Carol Carter + +Gadget149.50 +Gadget battery23.50 + + + diff --git a/integration-tests/xml/files/expected/0012-grouped.xsd b/integration-tests/xml/files/expected/0012-grouped.xsd new file mode 100644 index 0000000000..1d71132ba8 --- /dev/null +++ b/integration-tests/xml/files/expected/0012-grouped.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/files/expected/0013-multi-group-by.xml b/integration-tests/xml/files/expected/0013-multi-group-by.xml new file mode 100644 index 0000000000..53cf5a43df --- /dev/null +++ b/integration-tests/xml/files/expected/0013-multi-group-by.xml @@ -0,0 +1,31 @@ + + + +Alice Anderson + +29.99 +34.25 + + +Bob Brown + +119.95 +15.00 + + + + +Diane Davis + +114.00 +26.50 + + +Eric Evans + +42.50 +23.10 + + + + diff --git a/integration-tests/xml/files/expected/0013-multi-group-by.xsd b/integration-tests/xml/files/expected/0013-multi-group-by.xsd new file mode 100644 index 0000000000..0b2a26b58c --- /dev/null +++ b/integration-tests/xml/files/expected/0013-multi-group-by.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/files/expected/0014-document-fragment.xml b/integration-tests/xml/files/expected/0014-document-fragment.xml new file mode 100644 index 0000000000..b7e0554f37 --- /dev/null +++ b/integration-tests/xml/files/expected/0014-document-fragment.xml @@ -0,0 +1,6 @@ + + +Widget9.99120bluehardware +Gizmo19.95240red +Gadget49.50560graphitePremium edition. Special characters like & or < are preserved as text. + diff --git a/integration-tests/xml/files/expected/0014-document-fragment.xsd b/integration-tests/xml/files/expected/0014-document-fragment.xsd new file mode 100644 index 0000000000..d4d7f26522 --- /dev/null +++ b/integration-tests/xml/files/expected/0014-document-fragment.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/files/expected/0015-split_0_00001.xml b/integration-tests/xml/files/expected/0015-split_0_00001.xml new file mode 100644 index 0000000000..89bb5013c0 --- /dev/null +++ b/integration-tests/xml/files/expected/0015-split_0_00001.xml @@ -0,0 +1,8 @@ + + +start +checkpoint +checkpoint +checkpoint +checkpoint + diff --git a/integration-tests/xml/files/expected/0015-split_0_00002.xml b/integration-tests/xml/files/expected/0015-split_0_00002.xml new file mode 100644 index 0000000000..02f877997b --- /dev/null +++ b/integration-tests/xml/files/expected/0015-split_0_00002.xml @@ -0,0 +1,8 @@ + + +checkpoint +checkpoint +checkpoint +checkpoint +checkpoint + diff --git a/integration-tests/xml/files/expected/0015-split_0_00003.xml b/integration-tests/xml/files/expected/0015-split_0_00003.xml new file mode 100644 index 0000000000..2ca70a7f18 --- /dev/null +++ b/integration-tests/xml/files/expected/0015-split_0_00003.xml @@ -0,0 +1,5 @@ + + +checkpoint +end + diff --git a/integration-tests/xml/files/expected/0016-compact.xml b/integration-tests/xml/files/expected/0016-compact.xml new file mode 100644 index 0000000000..340b9b2bf2 --- /dev/null +++ b/integration-tests/xml/files/expected/0016-compact.xml @@ -0,0 +1 @@ +Widget9.99The blue widgetRefill0.00(no description)Gizmo19.95(no description) \ No newline at end of file diff --git a/integration-tests/xml/files/expected/0017-zipped.xml b/integration-tests/xml/files/expected/0017-zipped.xml new file mode 100644 index 0000000000..b756701d23 --- /dev/null +++ b/integration-tests/xml/files/expected/0017-zipped.xml @@ -0,0 +1,7 @@ + + +2026-01-151,250.00 EUR +2026-01-1587,500.50 EUR +2026-01-16250.75 EUR +2026-01-163,450.99 EUR + diff --git a/integration-tests/xml/files/expected/0017-zipped.xsd b/integration-tests/xml/files/expected/0017-zipped.xsd new file mode 100644 index 0000000000..75751b722f --- /dev/null +++ b/integration-tests/xml/files/expected/0017-zipped.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/main-0011-xml-output-advanced-basic.hwf b/integration-tests/xml/main-0011-xml-output-advanced-basic.hwf new file mode 100644 index 0000000000..708c3d6d5b --- /dev/null +++ b/integration-tests/xml/main-0011-xml-output-advanced-basic.hwf @@ -0,0 +1,210 @@ + + + + main-0011-xml-output-advanced-basic + Y + Integration test for the XML Output (Advanced) transform: minimal flat tree + sibling XSD. + + + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0011-xml-output-advanced-basic.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0011-xml-output-advanced-basic.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0011-basic.xml + ${PROJECT_HOME}/files/expected/0011-basic.xml + N + N + 544 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0011-basic.xsd + ${PROJECT_HOME}/files/expected/0011-basic.xsd + N + N + 720 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0011-basic.xml). + N + 544 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD does not match the expected output (${PROJECT_HOME}/files/expected/0011-basic.xsd). + N + 720 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0011-xml-output-advanced-basic.hpl + Y + Y + N + + + 0011-xml-output-advanced-basic.hpl + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0012-xml-output-advanced-grouped.hwf b/integration-tests/xml/main-0012-xml-output-advanced-grouped.hwf new file mode 100644 index 0000000000..98065c8020 --- /dev/null +++ b/integration-tests/xml/main-0012-xml-output-advanced-grouped.hwf @@ -0,0 +1,210 @@ + + + + main-0012-xml-output-advanced-grouped + Y + Integration test for the XML Output (Advanced) transform: group-by + namespace + DOCTYPE + XSL PI + XSD. + + + - + 2026/05/08 13:30:00.000 + - + 2026/05/08 13:30:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0012-xml-output-advanced-grouped.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0012-xml-output-advanced-grouped.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0012-grouped.xml + ${PROJECT_HOME}/files/expected/0012-grouped.xml + N + N + 544 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0012-grouped.xsd + ${PROJECT_HOME}/files/expected/0012-grouped.xsd + N + N + 720 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0012-grouped.xml). + N + 544 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD does not match the expected output (${PROJECT_HOME}/files/expected/0012-grouped.xsd). + N + 720 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0012-xml-output-advanced-grouped.hpl + Y + Y + N + + + 0012-xml-output-advanced-grouped.hpl + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0013-xml-output-advanced-multi-group-by.hwf b/integration-tests/xml/main-0013-xml-output-advanced-multi-group-by.hwf new file mode 100644 index 0000000000..afe5163671 --- /dev/null +++ b/integration-tests/xml/main-0013-xml-output-advanced-multi-group-by.hwf @@ -0,0 +1,210 @@ + + + + main-0013-xml-output-advanced-multi-group-by + Y + Integration test for the XML Output (Advanced) transform: two levels of group-by ancestors above the row loop. + + + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0013-xml-output-advanced-multi-group-by.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0013-xml-output-advanced-multi-group-by.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0013-multi-group-by.xml + ${PROJECT_HOME}/files/expected/0013-multi-group-by.xml + N + N + 544 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0013-multi-group-by.xsd + ${PROJECT_HOME}/files/expected/0013-multi-group-by.xsd + N + N + 720 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0013-multi-group-by.xml). + N + 544 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD does not match the expected output (${PROJECT_HOME}/files/expected/0013-multi-group-by.xsd). + N + 720 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0013-xml-output-advanced-multi-group-by.hpl + Y + Y + N + + + 0013-xml-output-advanced-multi-group-by.hpl + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0014-xml-output-advanced-document-fragment.hwf b/integration-tests/xml/main-0014-xml-output-advanced-document-fragment.hwf new file mode 100644 index 0000000000..7d476ca282 --- /dev/null +++ b/integration-tests/xml/main-0014-xml-output-advanced-document-fragment.hwf @@ -0,0 +1,210 @@ + + + + main-0014-xml-output-advanced-document-fragment + Y + Integration test for the XML Output (Advanced) transform: DocumentFragment node parses an XML field and inserts it as real XML. + + + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0014-xml-output-advanced-document-fragment.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0014-xml-output-advanced-document-fragment.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0014-document-fragment.xml + ${PROJECT_HOME}/files/expected/0014-document-fragment.xml + N + N + 544 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0014-document-fragment.xsd + ${PROJECT_HOME}/files/expected/0014-document-fragment.xsd + N + N + 720 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0014-document-fragment.xml). + N + 544 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD does not match the expected output (${PROJECT_HOME}/files/expected/0014-document-fragment.xsd). + N + 720 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0014-xml-output-advanced-document-fragment.hpl + Y + Y + N + + + 0014-xml-output-advanced-document-fragment.hpl + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0015-xml-output-advanced-split.hwf b/integration-tests/xml/main-0015-xml-output-advanced-split.hwf new file mode 100644 index 0000000000..0454c80912 --- /dev/null +++ b/integration-tests/xml/main-0015-xml-output-advanced-split.hwf @@ -0,0 +1,248 @@ + + + + main-0015-xml-output-advanced-split + Y + Integration test for the XML Output (Advanced) transform: split-every produces a sequence of files; verify each file's bytes. + + + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0015-xml-output-advanced-split.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0015-xml-output-advanced-split.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify split 1 + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0015-split_0_00001.xml + ${PROJECT_HOME}/files/expected/0015-split_0_00001.xml + N + N + 544 + 64 + + + + verify split 2 + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0015-split_0_00002.xml + ${PROJECT_HOME}/files/expected/0015-split_0_00002.xml + N + N + 720 + 64 + + + + verify split 3 + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0015-split_0_00003.xml + ${PROJECT_HOME}/files/expected/0015-split_0_00003.xml + N + N + 896 + 64 + + + + split 1 mismatch + + ABORT + + Generated split 1 does not match expected output (${PROJECT_HOME}/files/expected/0015-split_0_00001.xml). + N + 544 + 192 + + + + split 2 mismatch + + ABORT + + Generated split 2 does not match expected output (${PROJECT_HOME}/files/expected/0015-split_0_00002.xml). + N + 720 + 192 + + + + split 3 mismatch + + ABORT + + Generated split 3 does not match expected output (${PROJECT_HOME}/files/expected/0015-split_0_00003.xml). + N + 896 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0015-xml-output-advanced-split.hpl + Y + Y + N + + + 0015-xml-output-advanced-split.hpl + verify split 1 + Y + Y + N + + + verify split 1 + verify split 2 + Y + Y + N + + + verify split 2 + verify split 3 + Y + Y + N + + + verify split 1 + split 1 mismatch + Y + N + N + + + verify split 2 + split 2 mismatch + Y + N + N + + + verify split 3 + split 3 mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0016-xml-output-advanced-compact.hwf b/integration-tests/xml/main-0016-xml-output-advanced-compact.hwf new file mode 100644 index 0000000000..8681307c4f --- /dev/null +++ b/integration-tests/xml/main-0016-xml-output-advanced-compact.hwf @@ -0,0 +1,172 @@ + + + + main-0016-xml-output-advanced-compact + Y + Integration test for the XML Output (Advanced) transform: compact mode + null-value handling with force-create + create-attr-if-null/unmapped. + + + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0016-xml-output-advanced-compact.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0016-xml-output-advanced-compact.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0016-compact.xml + ${PROJECT_HOME}/files/expected/0016-compact.xml + N + N + 544 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0016-compact.xml). + N + 544 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0016-xml-output-advanced-compact.hpl + Y + Y + N + + + 0016-xml-output-advanced-compact.hpl + verify XML + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/main-0017-xml-output-advanced-zipped.hwf b/integration-tests/xml/main-0017-xml-output-advanced-zipped.hwf new file mode 100644 index 0000000000..7e67663e3a --- /dev/null +++ b/integration-tests/xml/main-0017-xml-output-advanced-zipped.hwf @@ -0,0 +1,262 @@ + + + + main-0017-xml-output-advanced-zipped + Y + Integration test for the XML Output (Advanced) transform: zipped output. The XML lives inside a .zip archive; the sibling .xsd is written next to the .zip. Verifies the inner XML by extracting it and comparing bytes, and verifies the sibling XSD directly. + + + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0017-xml-output-advanced-zipped.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0017-xml-output-advanced-zipped.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + create extracted + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output/extracted + N + N + 464 + 64 + + + + unzip archive + + UNZIP + + N + N + N + N + N + 0 + N + N + 1 + N + 10 + N + N + success_if_no_errors + ${java.io.tmpdir}/hop-xml-integration-output/extracted + ${java.io.tmpdir}/hop-xml-integration-output/0017-zipped.zip + N + 576 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/extracted/0017-zipped.xml + ${PROJECT_HOME}/files/expected/0017-zipped.xml + N + N + 704 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0017-zipped.xsd + ${PROJECT_HOME}/files/expected/0017-zipped.xsd + N + N + 848 + 64 + + + + XML mismatch + + ABORT + + Generated XML inside the zip does not match the expected output (${PROJECT_HOME}/files/expected/0017-zipped.xml). + N + 704 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD next to the zip does not match the expected output (${PROJECT_HOME}/files/expected/0017-zipped.xsd). + N + 848 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0017-xml-output-advanced-zipped.hpl + Y + Y + N + + + 0017-xml-output-advanced-zipped.hpl + create extracted + Y + Y + N + + + create extracted + unzip archive + Y + Y + N + + + unzip archive + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/integration-tests/xml/output/.gitignore b/integration-tests/xml/output/.gitignore new file mode 100644 index 0000000000..a8302c4f91 --- /dev/null +++ b/integration-tests/xml/output/.gitignore @@ -0,0 +1,3 @@ +# XML Output (Advanced) tests write under ${java.io.tmpdir}/hop-xml-integration-output; this dir is optional. +* +!.gitignore diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java index 645dc2febe..22d4b35f8a 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -49,7 +49,7 @@ import org.apache.hop.pipeline.transform.TransformMeta; /** - * Runtime engine for the Advanced XML Output transform. + * Runtime engine for the XML Output (Advanced) transform. * *

Walks the configured {@link XmlNode} tree once at first-row time, splitting it into a "prefix * path" (root → loop's parent) with optional group-by ancestors, a "loop subtree" emitted for each @@ -447,7 +447,7 @@ private void registerXsdResultFile(String xsdName) { ResultFile rf = new ResultFile( ResultFile.FILE_TYPE_GENERAL, xsd, getPipelineMeta().getName(), getTransformName()); - rf.setComment("XSD schema generated by Advanced XML Output transform"); + rf.setComment("XSD schema generated by XML Output (Advanced) transform"); addResultFile(rf); } catch (Exception e) { logError("Could not register sibling XSD as result file: " + e.getMessage()); @@ -551,7 +551,7 @@ private void registerResultFile() { data.currentFile, getPipelineMeta().getName(), getTransformName()); - rf.setComment("File created by Advanced XML Output transform"); + rf.setComment("File created by XML Output (Advanced) transform"); addResultFile(rf); } @@ -643,7 +643,8 @@ private void writeElement(XmlNode node, Object[] r) throws Exception { IValueMeta vm = data.inputRowMeta.getValueMeta(idx); Object data1 = r[idx]; value = vm.isNull(data1) ? null : vm.getString(data1); - } else if (!Utils.isEmpty(node.getDefaultValue())) { + } + if (value == null && !Utils.isEmpty(node.getDefaultValue())) { value = node.getDefaultValue(); } @@ -693,7 +694,20 @@ private void writeStartElementWithNamespace(XmlNode node) throws Exception { data.writer.writeDefaultNamespace(ns); } - /** Writes any direct attribute children of {@code node} for the current row. */ + /** + * Writes any direct attribute children of {@code node} for the current row. + * + *

Each attribute is emitted at most once. The decision tree is: + * + *

+ */ private void writeAttributesOf(XmlNode node, Object[] r) throws Exception { if (node.getChildren() == null) { return; @@ -702,40 +716,24 @@ private void writeAttributesOf(XmlNode node, Object[] r) throws Exception { if (c.getKind() != XmlNode.NodeKind.Attribute) { continue; } - Integer idx = data.fieldIndex.get(c); - String value = null; - if (idx != null) { - IValueMeta vm = data.inputRowMeta.getValueMeta(idx); - Object data1 = r[idx]; - value = vm.isNull(data1) ? null : vm.getString(data1); - } else if (!Utils.isEmpty(c.getDefaultValue())) { - value = c.getDefaultValue(); - } + String defaultVal = Utils.isEmpty(c.getDefaultValue()) ? "" : c.getDefaultValue(); - if (value == null) { - if (meta.isCreateAttributeIfNull() || c.isForceCreate()) { - data.writer.writeAttribute( - c.getName(), Utils.isEmpty(c.getDefaultValue()) ? "" : c.getDefaultValue()); + if (!data.fieldIndex.containsKey(c)) { + if (c.isForceCreate() || meta.isCreateAttributeIfUnmapped()) { + data.writer.writeAttribute(c.getName(), defaultVal); } - // else: skip silently - } else { - data.writer.writeAttribute(c.getName(), applyTrim(value)); + continue; } - } - // Unmapped attributes that the user wants to emit anyway (rare; v1 covers via forceCreate). - if (meta.isCreateAttributeIfUnmapped() && node.getChildren() != null) { - for (XmlNode c : node.getChildren()) { - if (c.getKind() != XmlNode.NodeKind.Attribute) { - continue; - } - if (data.fieldIndex.containsKey(c)) { - continue; // already handled - } - if (Utils.isEmpty(c.getMappedField()) && !c.isForceCreate()) { - // forceCreate path is already covered above with default value - data.writer.writeAttribute( - c.getName(), Utils.isEmpty(c.getDefaultValue()) ? "" : c.getDefaultValue()); - } + + int idx = data.fieldIndex.get(c); + IValueMeta vm = data.inputRowMeta.getValueMeta(idx); + Object data1 = r[idx]; + String value = vm.isNull(data1) ? null : vm.getString(data1); + + if (value != null) { + data.writer.writeAttribute(c.getName(), applyTrim(value)); + } else if (c.isForceCreate() || meta.isCreateAttributeIfNull()) { + data.writer.writeAttribute(c.getName(), defaultVal); } } } @@ -759,7 +757,8 @@ private void writeOwnTextValue(XmlNode node, Object[] r) throws Exception { IValueMeta vm = data.inputRowMeta.getValueMeta(idx); Object data1 = r[idx]; value = vm.isNull(data1) ? null : vm.getString(data1); - } else if (!Utils.isEmpty(node.getDefaultValue())) { + } + if (value == null && !Utils.isEmpty(node.getDefaultValue())) { value = node.getDefaultValue(); } if (value != null) { diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index bfbb6bfbb7..f9d989ac03 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -53,7 +53,7 @@ import org.eclipse.swt.widgets.Text; /** - * Dialog for the Advanced XML Output transform. + * Dialog for the XML Output (Advanced) transform. * *

Three tabs: File (filename, encoding, splitting, zipping), Content (XML * declaration, formatting, doctype, xsl, xsd) and XML Tree (visual designer for the diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java index c28e078edb..0d50889011 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -41,7 +41,7 @@ import org.apache.hop.resource.ResourceDefinition; /** - * Metadata for the Advanced XML Output transform: writes input rows to one or more XML files + * Metadata for the XML Output (Advanced) transform: writes input rows to one or more XML files * following a hierarchical, user-defined XML tree (with one row-loop element and optional group-by * ancestors). */ @@ -52,7 +52,7 @@ description = "i18n::AdvancedXMLOutput.description", categoryDescription = "i18n::AdvancedXMLOutput.category", keywords = "i18n::AdvancedXmlOutputMeta.keyword", - documentationUrl = "/pipeline/transforms/advancedxmloutput.html", + documentationUrl = "/pipeline/transforms/xmloutputadvanced.html", isIncludeJdbcDrivers = false) @Getter @Setter diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java index 595f711d6e..a738027b02 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java @@ -27,7 +27,7 @@ import org.apache.hop.metadata.api.HopMetadataProperty; /** - * Describes a single node in the hierarchical XML tree of the Advanced XML Output transform. + * Describes a single node in the hierarchical XML tree of the XML Output (Advanced) transform. * *

An {@code XmlNode} can represent an element or an attribute. Elements may have any number of * child nodes (recursive structure). Exactly one element in the tree must be marked as the loop diff --git a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties index 95bebc2ff8..b27782c213 100644 --- a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties +++ b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties @@ -15,7 +15,7 @@ # limitations under the License. # -AdvancedXMLOutput.name=Advanced XML output +AdvancedXMLOutput.name=XML Output (Advanced) AdvancedXMLOutput.description=Write rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. AdvancedXMLOutput.category=Output AdvancedXmlOutputMeta.keyword=xml,output,write,tree,hierarchical,advanced @@ -55,7 +55,7 @@ AdvancedXMLOutputMeta.CheckResult.ExpectedInputError=No input received from prev # ------------------------------------------------------------ # Dialog # ------------------------------------------------------------ -AdvancedXMLOutputDialog.Shell.Title=Advanced XML output +AdvancedXMLOutputDialog.Shell.Title=XML Output (Advanced) AdvancedXMLOutputDialog.TransformName.Label=Transform name AdvancedXMLOutputDialog.ErrorGettingFields=Unable to read fields from the previous transform. AdvancedXMLOutputDialog.ErrorGettingFields.Title=Field lookup failed diff --git a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-basic.hpl similarity index 94% rename from plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl rename to plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-basic.hpl index fa6fbd981e..1275bd4fbe 100644 --- a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-basic.hpl +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-basic.hpl @@ -19,9 +19,9 @@ limitations under the License. --> - advanced-xml-output-basic + xml-output-advanced-basic Y - Minimal Advanced XML Output example: emit a flat <customers><customer>... document with a sibling .xsd schema. + Minimal XML Output (Advanced) example: emit a flat <customers><customer>... document with a sibling .xsd schema. Normal @@ -56,14 +56,14 @@ limitations under the License. Writes a flat <customers><customer>... document together with a sibling .xsd describing the structure. -Output: ${java.io.tmpdir}/advanced-xml-output-basic.xml (+ .xsd) +Output: ${java.io.tmpdir}/xml-output-advanced-basic.xml (+ .xsd) 540 customers grid - Advanced XML output + XML Output (Advanced) Y @@ -172,7 +172,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-basic.xml (+ .xsd) - Advanced XML output + XML Output (Advanced) AdvancedXMLOutput Pretty-prints the rows as <customer> elements under a <customers> root, alongside a generated XSD. Y @@ -196,7 +196,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-basic.xml (+ .xsd) text/xsl - ${java.io.tmpdir}/advanced-xml-output-basic + ${java.io.tmpdir}/xml-output-advanced-basic xml 0 N diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-compact.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-compact.hpl new file mode 100644 index 0000000000..eb520511cb --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-compact.hpl @@ -0,0 +1,270 @@ + + + + + xml-output-advanced-compact + Y + Compact output combined with force-create, default values, and the create-attribute-if-null / create-attribute-if-unmapped flags. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 148 + 32 + 16 + This sample exercises the null-handling and unmapped flags: + + * @currency has no mapped field at all but is still emitted, with its + default_value, because Create attribute if unmapped is on. + * @discontinued is force-created with default_value="false" when + its mapped field is null. + * <note> is force-created with a default text value. + * Compact mode strips whitespace between elements. + +Output: ${java.io.tmpdir}/xml-output-advanced-compact.xml + 700 + + + + + products grid + XML Output (Advanced) + Y + + + + products grid + DataGrid + Three products. Some rows leave optional columns NULL to exercise force-create + default values. + Y + + 1 + + none + + + + + SKU-100 + Widget + 9.99 + The blue widget + false + + + SKU-150 + Refill + + + + + + SKU-200 + Gizmo + 19.95 + + true + + + + + -1 + -1 + + N + sku + + + + String + + + -1 + -1 + + N + name + + + + String + + + -1 + 2 + + N + price + 0.00 + + . + Number + + + -1 + -1 + + N + note + + + + String + + + -1 + -1 + + N + discontinued + + + + Boolean + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Compact + create-attr-if-null + create-attr-if-unmapped + force-create with default values. + Y + + 1 + + none + + + UTF-8 + Y + N + Y + Y + Y + Y + N + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-compact + xml + 0 + N + N + N + N + Y + N + Y + N + + + + catalog + Element + + + product + Element + Y + + + sku + Attribute + sku + + + currency + Attribute + USD + + + discontinued + Attribute + discontinued + false + Y + + + name + Element + name + + + price + Element + price + 0.00 + 0.00 + Y + + + note + Element + note + (no description) + Y + + + + + + + + 320 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-document-fragment.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-document-fragment.hpl new file mode 100644 index 0000000000..b53c4de377 --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-document-fragment.hpl @@ -0,0 +1,236 @@ + + + + + xml-output-advanced-document-fragment + Y + DocumentFragment node example: an input field holds a well-formed XML fragment that is parsed and inserted as XML rather than escaped text. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 120 + 32 + 16 + The "extras" field holds a pre-built XML fragment per row. The +DocumentFragment node parses it and inserts the result as real XML +nodes (not escaped text). Multiple top-level fragment elements are +supported - they are written as siblings under the parent <product>. + +Output: ${java.io.tmpdir}/xml-output-advanced-document-fragment.xml + 620 + + + + + products grid + XML Output (Advanced) + Y + + + + products grid + DataGrid + Three products, each with its own pre-built XML fragment of metadata. + Y + + 1 + + none + + + + + SKU-100 + Widget + 9.99 + <weight unit="g">120</weight><color>blue</color><tag>hardware</tag> + + + SKU-200 + Gizmo + 19.95 + <weight unit="g">240</weight><color>red</color><dimensions w="10" h="5" d="3"/> + + + SKU-300 + Gadget + 49.50 + <weight unit="g">560</weight><color>graphite</color><notes priority="low">Premium edition. Special characters like &amp; or &lt; are preserved as text.</notes> + + + + + -1 + -1 + + N + sku + + + + String + + + -1 + -1 + + N + name + + + + String + + + -1 + 2 + + N + price + 0.00 + + . + Number + + + -1 + -1 + + N + extras + + + + String + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + products/product(loop) with an Element name and a DocumentFragment that injects the "extras" field as XML. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-document-fragment + xml + 0 + N + N + N + N + Y + N + Y + N + + + + products + Element + + + product + Element + Y + + + sku + Attribute + sku + + + name + Element + name + + + price + Element + price + 0.00 + + + extras + DocumentFragment + extras + + + + + + + + 320 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-grouped.hpl similarity index 95% rename from plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl rename to plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-grouped.hpl index 45c930cea4..575ed5656c 100644 --- a/plugins/transforms/xml/src/main/samples/transforms/advanced-xml-output-grouped.hpl +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-grouped.hpl @@ -19,9 +19,9 @@ limitations under the License. --> - advanced-xml-output-grouped + xml-output-advanced-grouped Y - Advanced XML Output example with group-by, attributes, a default XML namespace, a DOCTYPE, an XSL stylesheet processing instruction, and a generated XSD. + XML Output (Advanced) example with group-by, attributes, a default XML namespace, a DOCTYPE, an XSL stylesheet processing instruction, and a generated XSD. Normal @@ -62,7 +62,7 @@ The transform also emits: - an <?xml-stylesheet?> processing instruction, - a sibling .xsd schema next to the .xml output. -Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) +Output: ${java.io.tmpdir}/xml-output-advanced-grouped.xml (+ .xsd) 620 @@ -74,7 +74,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) sort by orderId - Advanced XML output + XML Output (Advanced) Y @@ -235,7 +235,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) ${java.io.tmpdir} - advanced-xml-output-grouped-sort + xml-output-advanced-grouped-sort 1000000 N @@ -248,7 +248,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) - Advanced XML output + XML Output (Advanced) AdvancedXMLOutput orders/order/lines/line tree with group-by on orderId and loop on line. Y @@ -274,7 +274,7 @@ Output: ${java.io.tmpdir}/advanced-xml-output-grouped.xml (+ .xsd) orders.xsl text/xsl - ${java.io.tmpdir}/advanced-xml-output-grouped + ${java.io.tmpdir}/xml-output-advanced-grouped xml 0 N diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-multi-group-by.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-multi-group-by.hpl new file mode 100644 index 0000000000..7ac6cba58b --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-multi-group-by.hpl @@ -0,0 +1,385 @@ + + + + + xml-output-advanced-multi-group-by + Y + Two levels of group-by: rows are folded under <region> then under <order>, with line items as the row loop. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 132 + 32 + 16 + Two levels of group-by demonstrate that the loop can sit inside multiple +group-by ancestors. The Sort Rows step pre-sorts by region, then orderId, so +the runtime can collapse consecutive matching keys without buffering. + +Tree: regions/region[group region] / order[group orderId] / lines/line[loop] + +Output: ${java.io.tmpdir}/xml-output-advanced-multi-group-by.xml + 620 + + + + + order lines grid + sort by region, orderId + Y + + + sort by region, orderId + XML Output (Advanced) + Y + + + + order lines grid + DataGrid + Two regions, four orders, eight line items. + Y + + 1 + + none + + + + + EU + 1001 + Alice Anderson + SKU-100 + 2 + 9.99 + + + NA + 2001 + Diane Davis + SKU-510 + 1 + 14.00 + + + EU + 1001 + Alice Anderson + SKU-150 + 3 + 4.25 + + + EU + 1002 + Bob Brown + SKU-200 + 1 + 19.95 + + + NA + 2002 + Eric Evans + SKU-600 + 4 + 2.50 + + + EU + 1002 + Bob Brown + SKU-250 + 1 + 5.00 + + + NA + 2002 + Eric Evans + SKU-610 + 2 + 3.10 + + + NA + 2001 + Diane Davis + SKU-520 + 2 + 6.50 + + + + + -1 + -1 + + N + region + + + + String + + + -1 + -1 + + N + orderId + + + + Integer + + + -1 + -1 + + N + customer + + + + String + + + -1 + -1 + + N + productSku + + + + String + + + -1 + -1 + + N + qty + + + + Integer + + + -1 + 2 + + N + unitPrice + 0.00 + + . + Number + + + + + 96 + 208 + + + + sort by region, orderId + SortRows + Group-by needs sorted input. Sort by every group key from outermost to innermost. + Y + + 1 + + none + + + + + region + Y + N + N + 0 + N + + + orderId + Y + N + N + 0 + N + + + ${java.io.tmpdir} + xml-output-advanced-multi-group-by-sort + 1000000 + + N + N + + + + 320 + 208 + + + + XML Output (Advanced) + AdvancedXMLOutput + regions/region(group)/order(group)/lines/line(loop) with attribute @code on the region. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + Y + Y + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-multi-group-by + xml + 0 + N + N + N + N + Y + N + Y + N + + + + regions + Element + + + region + Element + Y + region + + + code + Attribute + region + + + order + Element + Y + orderId + + + id + Attribute + orderId + + + customer + Element + customer + + + lines + Element + Y + + + line + Element + Y + + + sku + Attribute + productSku + + + qty + Element + qty + + + price + Element + unitPrice + 0.00 + + + + + + + + + + + + + + 544 + 208 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-split.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-split.hpl new file mode 100644 index 0000000000..643ec69eaa --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-split.hpl @@ -0,0 +1,197 @@ + + + + + xml-output-advanced-split + Y + Roll over to a new output file every N input rows; the transform copy number and a 5-digit split sequence are appended to the filename. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 120 + 32 + 16 + Twelve events are emitted as five-row splits: three files in total. Each +file is closed and a fresh XML prolog/root is opened on the next split. + +Filename pattern: xml-output-advanced-split_{copyNr}_{00001..} .xml + +Output: ${java.io.tmpdir}/xml-output-advanced-split_0_00001.xml (etc.) + 620 + + + + + events grid + XML Output (Advanced) + Y + + + + events grid + DataGrid + Twelve numbered events. + Y + + 1 + + none + + + + 1start + 2checkpoint + 3checkpoint + 4checkpoint + 5checkpoint + 6checkpoint + 7checkpoint + 8checkpoint + 9checkpoint + 10checkpoint + 11checkpoint + 12end + + + + -1 + -1 + + N + eventId + + + + Integer + + + -1 + -1 + + N + kind + + + + String + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Split every 5 rows; include the transform copy number in the filename. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-split + xml + 5 + Y + N + N + N + Y + N + Y + N + + + + events + Element + + + event + Element + Y + + + id + Attribute + eventId + + + kind + Element + kind + + + + + + + + 320 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-zipped.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-zipped.hpl new file mode 100644 index 0000000000..f107149b78 --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-zipped.hpl @@ -0,0 +1,211 @@ + + + + + xml-output-advanced-zipped + Y + Wraps the output XML in a zip archive and stamps a custom date/time pattern into the archive name. The sibling XSD is written next to the .zip, not inside it. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 14:50:00.000 + - + 2026/05/08 14:50:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 120 + 32 + 16 + The "specify format" toggle replaces the date/time toggles with a free-form +SimpleDateFormat pattern. Combined with zipped output, the archive lands at: + + ${java.io.tmpdir}/transactions-yyyy-MM-dd_HH-mm-ss.zip + +Inside the zip, one entry "transactions-...xml" holds the data. The sibling +.xsd is written next to the .zip, never inside it. + 620 + + + + + transactions grid + XML Output (Advanced) + Y + + + + transactions grid + DataGrid + A handful of monetary transactions with a EUR currency symbol and grouped thousands. + Y + + 1 + + none + + + + TX-10012026-01-151250.00 + TX-10022026-01-1587500.50 + TX-10032026-01-16250.75 + TX-10042026-01-163450.99 + + + + -1 + -1 + + N + txId + + + + String + + + -1 + -1 + + N + txDate + yyyy-MM-dd + + + Date + + + -1 + 2 + + N + amount + 0.00 + + . + Number + + + + + 96 + 192 + + + + XML Output (Advanced) + AdvancedXMLOutput + Per-node currency, decimal and grouping symbols on the <amount> element. + Y + + 1 + + none + + + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/transactions- + xml + 0 + N + N + N + Y + Y + N + Y + Y + yyyy-MM-dd_HH-mm-ss + + + transactions + Element + + + transaction + Element + Y + + + id + Attribute + txId + + + date + Element + txDate + yyyy-MM-dd + + + amount + Element + amount + #,##0.00 ¤ + EUR + . + , + + + + + + + + 320 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java index 29f48158b5..b7ba04ffaa 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java @@ -18,6 +18,7 @@ package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -45,6 +46,9 @@ */ class AdvancedXmlOutputSamplesTest { + /** All sample pipelines that demonstrate the XML Output (Advanced) transform. */ + private static final String SAMPLE_PREFIX = "xml-output-advanced-"; + private static final Path SAMPLES_DIR = Path.of("src", "main", "samples", "transforms").toAbsolutePath(); @@ -55,10 +59,10 @@ static void setup() throws Exception { @Test void basicSampleParsesAndHasExpectedShape() throws Exception { - AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("advanced-xml-output-basic.hpl"); + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-basic.hpl"); assertEquals( - "${java.io.tmpdir}/advanced-xml-output-basic", meta.getFileSupport().getFileName()); + "${java.io.tmpdir}/xml-output-advanced-basic", meta.getFileSupport().getFileName()); assertEquals("xml", meta.getFileSupport().getExtension()); assertTrue(meta.isGenerateXsd(), "basic sample is meant to demonstrate XSD generation"); @@ -77,7 +81,7 @@ void basicSampleParsesAndHasExpectedShape() throws Exception { @Test void groupedSampleParsesAndHasExpectedShape() throws Exception { - AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("advanced-xml-output-grouped.hpl"); + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-grouped.hpl"); assertTrue(meta.isGenerateXsd()); assertEquals("orders", meta.getDoctypeRootElement()); @@ -113,13 +117,121 @@ void groupedSampleParsesAndHasExpectedShape() throws Exception { } @Test - void allSamplesAreWellFormedXml() throws Exception { + void multiGroupBySampleHasTwoNestedGroupAncestors() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-multi-group-by.hpl"); + + XmlNode root = meta.getRootNode(); + assertEquals("regions", root.getName()); + + XmlNode region = root.getChildren().get(0); + assertEquals("region", region.getName()); + assertTrue(region.isGroupBy(), " must be flagged group-by (outer)"); + assertEquals("region", region.getMappedField()); + + // region > [code @, order] + XmlNode order = + region.getChildren().stream() + .filter(c -> "order".equals(c.getName())) + .findFirst() + .orElseThrow(); + assertTrue(order.isGroupBy(), " must be flagged group-by (inner)"); + assertEquals("orderId", order.getMappedField()); + + // Find the loop element down the tree. + XmlNode loop = findLoop(root); + assertNotNull(loop, "the multi-group-by sample must declare exactly one loop element"); + assertEquals("line", loop.getName()); + } + + @Test + void documentFragmentSampleEmbedsFragmentNode() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-document-fragment.hpl"); + + XmlNode root = meta.getRootNode(); + assertEquals("products", root.getName()); + XmlNode product = root.getChildren().get(0); + assertEquals("product", product.getName()); + assertTrue(product.isLoop()); + + boolean foundFragment = false; + for (XmlNode c : product.getChildren()) { + if (c.getKind() == XmlNode.NodeKind.DocumentFragment) { + assertEquals("extras", c.getMappedField()); + foundFragment = true; + } + } + assertTrue( + foundFragment, "document-fragment sample must include at least one DocumentFragment node"); + } + + @Test + void splitSampleEnablesSplitEveryAndTransformNr() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-split.hpl"); + + XmlFileOutputSupport file = meta.getFileSupport(); + assertEquals(5, file.getSplitEvery(), " drives split rollover"); + assertTrue(file.isTransformNrInFilename(), "the copy number must be added to the filename"); + assertFalse(file.isZipped()); + assertFalse(meta.isGenerateXsd(), "the split sample intentionally turns XSD off"); + } + + @Test + void zippedSampleUsesCustomDateTimePatternAndZip() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-zipped.hpl"); + + XmlFileOutputSupport file = meta.getFileSupport(); + assertTrue(file.isZipped(), " must be on"); + assertTrue(file.isSpecifyFormat(), " must be on"); + assertEquals("yyyy-MM-dd_HH-mm-ss", file.getDateTimeFormat()); + assertTrue(meta.isGenerateXsd(), "the XSD lives next to the .zip, not inside it"); + } + + @Test + void compactSampleExercisesNullHandlingFlags() throws Exception { + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-compact.hpl"); + + assertTrue(meta.isCompactFile(), " must be on"); + assertFalse(meta.isBlankLineAfterXmlDeclaration()); + assertTrue(meta.isCreateAttributeIfNull()); + assertTrue(meta.isCreateAttributeIfUnmapped()); + + XmlNode product = meta.getRootNode().getChildren().get(0); + long forceCreated = product.getChildren().stream().filter(XmlNode::isForceCreate).count(); + assertTrue(forceCreated >= 2, "compact sample must have at least two force-create children"); + + long withDefaults = + product.getChildren().stream() + .filter(c -> c.getDefaultValue() != null && !c.getDefaultValue().isEmpty()) + .count(); + assertTrue(withDefaults >= 3, "compact sample must define at least three default values"); + + boolean foundUnmappedAttr = + product.getChildren().stream() + .anyMatch( + c -> + c.getKind() == XmlNode.NodeKind.Attribute + && (c.getMappedField() == null || c.getMappedField().isEmpty()) + && c.getDefaultValue() != null + && !c.getDefaultValue().isEmpty()); + assertTrue( + foundUnmappedAttr, + "compact sample must have at least one unmapped attribute with a default value"); + } + + @Test + void allSamplesAreWellFormedXmlAndDeserializeCleanly() throws Exception { List samples = listSamples(); - assertTrue(samples.size() >= 2, "expected at least two sample pipelines"); + assertTrue(samples.size() >= 7, "expected the seven shipped sample pipelines"); for (Path p : samples) { Document doc = XmlHandler.loadXmlString(Files.readString(p)); assertNotNull(doc, "could not parse " + p.getFileName()); assertEquals("pipeline", doc.getDocumentElement().getNodeName(), "wrong root in " + p); + + // Every shipped sample must contain exactly one AdvancedXMLOutput transform + // that round-trips through metadata serialization without errors. + AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput(p.getFileName().toString()); + assertNotNull(meta.getRootNode(), "no root tree in " + p.getFileName()); + assertNotNull(findLoop(meta.getRootNode()), "no loop element in " + p.getFileName()); } } @@ -155,10 +267,29 @@ private static List listSamples() throws Exception { try (var stream = Files.list(SAMPLES_DIR)) { stream .filter(p -> p.getFileName().toString().endsWith(".hpl")) - .filter(p -> p.getFileName().toString().startsWith("advanced-xml-output-")) + .filter(p -> p.getFileName().toString().startsWith(SAMPLE_PREFIX)) .sorted() .forEach(out::add); } return out; } + + private static XmlNode findLoop(XmlNode node) { + if (node == null) { + return null; + } + if (node.isLoop()) { + return node; + } + if (node.getChildren() == null) { + return null; + } + for (XmlNode c : node.getChildren()) { + XmlNode hit = findLoop(c); + if (hit != null) { + return hit; + } + } + return null; + } } diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java index c42563eea6..6d17767709 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java @@ -41,8 +41,8 @@ import org.junit.jupiter.api.io.TempDir; /** - * End-to-end runtime tests for the Advanced XML Output transform. Each test runs an actual pipeline - * via the local engine, writes to a temp file and asserts on the produced XML. + * End-to-end runtime tests for the XML Output (Advanced) transform. Each test runs an actual + * pipeline via the local engine, writes to a temp file and asserts on the produced XML. */ class AdvancedXmlOutputTest { @@ -210,6 +210,60 @@ void testForceCreateEmitsStaticElementWithDefaultValue(@TempDir Path tempDir) th assertTrue(xml.contains("(none)"), "expected force-created element: " + xml); } + /** + * A mapped element whose row value is null and that has both {@code force_create} and a {@code + * default_value} should fall back to the default value, not to an empty tag. + */ + @Test + void testMappedElementFallsBackToDefaultWhenRowValueIsNull(@TempDir Path tempDir) + throws Exception { + Path output = tempDir.resolve("nullfallback"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + // already has mapped to "name". Mark it force-create + default and feed a null. + XmlNode loop = meta.getRootNode().getChildren().get(0); + XmlNode name = loop.getChildren().get(0); + name.setForceCreate(true); + name.setDefaultValue("(unknown)"); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("name")); + rm.addValueMeta(new ValueMetaInteger("age")); + List rows = new ArrayList<>(); + rows.add(new RowMetaAndData(rm, null, 30L)); + + runPipeline(meta, rows); + + String xml = readWrittenFile(output); + assertTrue( + xml.contains("(unknown)"), + "mapped element with null row value must fall back to its default value: " + xml); + } + + /** + * An attribute with no mapped field but a non-empty default value must be emitted exactly once, + * even when {@code Create attribute if no field is mapped} is on. (Regression: an earlier + * implementation walked attribute children twice and produced a duplicated attribute.) + */ + @Test + void testUnmappedAttributeWithDefaultIsEmittedExactlyOnce(@TempDir Path tempDir) + throws Exception { + Path output = tempDir.resolve("unmappedonce"); + AdvancedXmlOutputMeta meta = buildFlatMeta(output.toString()); + meta.setCreateAttributeIfUnmapped(true); + + XmlNode loop = meta.getRootNode().getChildren().get(0); + XmlNode currency = new XmlNode("currency", XmlNode.NodeKind.Attribute); + currency.setDefaultValue("USD"); + loop.addChild(currency); + + runPipeline(meta, buildFlatRows("Alice", 30)); + + String xml = readWrittenFile(output); + int matches = count(xml, "currency=\"USD\""); + assertEquals( + 1, matches, "unmapped attribute with default value must be emitted exactly once: " + xml); + } + // --------------------------------------------------------------------------- // Split-every produces multiple files // --------------------------------------------------------------------------- From 2373acb1def1fdc25f88f152a63c9c0684bcc45e Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 14 May 2026 10:44:08 +0200 Subject: [PATCH 4/9] dialog updates --- .../AdvancedXmlOutputDialog.java | 91 ++++++++----------- .../advancedxmloutput/XmlTreeDesigner.java | 6 +- 2 files changed, 44 insertions(+), 53 deletions(-) diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index f9d989ac03..daaf88f44f 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -38,9 +38,11 @@ import org.eclipse.swt.custom.CCombo; import org.eclipse.swt.custom.CTabFolder; import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.custom.ScrolledComposite; import org.eclipse.swt.events.ModifyListener; import org.eclipse.swt.events.SelectionAdapter; import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.layout.FillLayout; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; @@ -114,46 +116,32 @@ public AdvancedXmlOutputDialog( @Override public String open() { - Shell parent = getParent(); - shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); - PropsUi.setLook(shell); - setShellImage(shell, input); - - ModifyListener lsMod = e -> input.setChanged(); + createShell(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Shell.Title")); changed = input.hasChanged(); - FormLayout formLayout = new FormLayout(); - formLayout.marginWidth = PropsUi.getFormMargin(); - formLayout.marginHeight = PropsUi.getFormMargin(); - shell.setLayout(formLayout); - shell.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Shell.Title")); - - int margin = PropsUi.getMargin(); - int middle = props.getMiddlePct(); - - // Transform name - Label wlTransformName = new Label(shell, SWT.RIGHT); - wlTransformName.setText( - BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.TransformName.Label")); - PropsUi.setLook(wlTransformName); - fdlTransformName = new FormData(); - fdlTransformName.left = new FormAttachment(0, 0); - fdlTransformName.top = new FormAttachment(0, margin); - fdlTransformName.right = new FormAttachment(middle, -margin); - wlTransformName.setLayoutData(fdlTransformName); - - wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - wTransformName.setText(transformName); - PropsUi.setLook(wTransformName); - wTransformName.addModifyListener(lsMod); - fdTransformName = new FormData(); - fdTransformName.left = new FormAttachment(middle, 0); - fdTransformName.top = new FormAttachment(0, margin); - fdTransformName.right = new FormAttachment(100, 0); - wTransformName.setLayoutData(fdTransformName); - - // Tabs - CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + + ScrolledComposite wScrolledComposite = + new ScrolledComposite(shell, SWT.V_SCROLL | SWT.H_SCROLL); + PropsUi.setLook(wScrolledComposite); + FormData fdSc = new FormData(); + fdSc.left = new FormAttachment(0, 0); + fdSc.top = new FormAttachment(wSpacer, 0); + fdSc.right = new FormAttachment(100, 0); + fdSc.bottom = new FormAttachment(wOk, -margin); + wScrolledComposite.setLayoutData(fdSc); + wScrolledComposite.setLayout(new FillLayout()); + wScrolledComposite.setExpandHorizontal(true); + wScrolledComposite.setExpandVertical(true); + + Composite mainComposite = new Composite(wScrolledComposite, SWT.NONE); + PropsUi.setLook(mainComposite); + mainComposite.setLayout(props.createFormLayout()); + + Label wContentTop = new Label(mainComposite, SWT.NONE); + wContentTop.setLayoutData(new FormData(0, 0)); + + CTabFolder wTabFolder = new CTabFolder(mainComposite, SWT.BORDER); PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); addFileTab(wTabFolder, lsMod, margin, middle); @@ -162,20 +150,23 @@ public String open() { FormData fdTabFolder = new FormData(); fdTabFolder.left = new FormAttachment(0, 0); - fdTabFolder.top = new FormAttachment(wTransformName, margin); fdTabFolder.right = new FormAttachment(100, 0); - fdTabFolder.bottom = new FormAttachment(100, -50); + fdTabFolder.top = new FormAttachment(wContentTop, margin); + fdTabFolder.bottom = new FormAttachment(100, -margin); wTabFolder.setLayoutData(fdTabFolder); - // Buttons - wOk = new Button(shell, SWT.PUSH); - wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); - wOk.addListener(SWT.Selection, e -> ok()); - wCancel = new Button(shell, SWT.PUSH); - wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); - wCancel.addListener(SWT.Selection, e -> cancel()); + wScrolledComposite.setContent(mainComposite); + mainComposite.pack(); + wScrolledComposite.setMinSize(mainComposite.computeSize(SWT.DEFAULT, SWT.DEFAULT)); + + FormData fdComp = new FormData(); + fdComp.left = new FormAttachment(0, 0); + fdComp.top = new FormAttachment(0, 0); + fdComp.right = new FormAttachment(100, 0); + fdComp.bottom = new FormAttachment(100, 0); + mainComposite.setLayoutData(fdComp); - setButtonPositions(new Button[] {wOk, wCancel}, margin, null); + mainComposite.pack(); wTabFolder.setSelection(0); @@ -186,6 +177,7 @@ public String open() { input.setChanged(changed); + focusTransformName(); BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); return transformName; @@ -747,9 +739,6 @@ public void getData() { XmlNode root = input.getRootNode() != null ? new XmlNode(input.getRootNode()) : defaultRootNode(); wTreeDesigner.setRootNode(root); - - wTransformName.selectAll(); - wTransformName.setFocus(); } private static XmlNode defaultRootNode() { diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java index d80a3fc113..51d99bf0ff 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java @@ -299,13 +299,15 @@ private void buildPropertiesPane(Composite parent) { layoutValueRight(lblKind, wpKind, labelWidth); wpKind.addModifyListener(e -> applyPropertiesToModel()); - Label lblField = addLabel(props, lblKind, "MappedField", labelWidth); + // Attach each row below the previous row's value control (not the label): CCombo is taller than + // a Label, so attaching the next label under the previous label caused vertical overlap. + Label lblField = addLabel(props, wpKind, "MappedField", labelWidth); wpMappedField = new CCombo(props, SWT.BORDER); PropsUi.setLook(wpMappedField); layoutValueRight(lblField, wpMappedField, labelWidth); wpMappedField.addModifyListener(e -> applyPropertiesToModel()); - wpDefaultValue = addLabeledText(props, lblField, "DefaultValue", labelWidth); + wpDefaultValue = addLabeledText(props, wpMappedField, "DefaultValue", labelWidth); wpFormat = addLabeledText(props, wpDefaultValue, "Format", labelWidth); Composite numericRow = new Composite(props, SWT.NONE); From 445d202be45ea02fb5fe2b5b45b32ca644255d45 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 14 May 2026 14:55:25 +0200 Subject: [PATCH 5/9] minor tweaks and updates, updated integration tests and docs --- .../ROOT/pages/pipeline/transforms.adoc | 1 + .../transforms/xmloutputadvanced.adoc | 16 +- .../xml/0018-xml-output-advanced-chained.hpl | 364 +++++++++++++++++ .../xml/files/expected/0018-chained.xml | 21 + .../xml/files/expected/0018-chained.xsd | 2 + .../main-0018-xml-output-advanced-chained.hwf | 210 ++++++++++ .../advancedxmloutput/AdvancedXmlOutput.java | 226 +++++++++-- .../AdvancedXmlOutputData.java | 21 + .../AdvancedXmlOutputDialog.java | 133 ++++++- .../AdvancedXmlOutputMeta.java | 156 +++++++- .../xml/advancedxmloutput/XmlNode.java | 12 + .../advancedxmloutput/XmlTreeDesigner.java | 16 + .../messages/messages_en_US.properties | 19 +- .../xml-output-advanced-chained.hpl | 364 +++++++++++++++++ .../xml-output-advanced-person-addresses.hpl | 368 ++++++++++++++++++ ...OutputAdvanced0018GoldenGeneratorTest.java | 242 ++++++++++++ .../AdvancedXmlOutputMetaTest.java | 39 ++ .../AdvancedXmlOutputSamplesTest.java | 123 +++++- .../AdvancedXmlOutputTest.java | 235 +++++++++++ 19 files changed, 2520 insertions(+), 48 deletions(-) create mode 100644 integration-tests/xml/0018-xml-output-advanced-chained.hpl create mode 100644 integration-tests/xml/files/expected/0018-chained.xml create mode 100644 integration-tests/xml/files/expected/0018-chained.xsd create mode 100644 integration-tests/xml/main-0018-xml-output-advanced-chained.hwf create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-chained.hpl create mode 100644 plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-person-addresses.hpl create mode 100644 plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc index 09d9d4b4b8..4933b31be7 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc @@ -242,6 +242,7 @@ The pages nested under this topic contain information on how to use the transfor * xref:pipeline/transforms/xmlinputstream.adoc[XML Input Stream (StAX)] * xref:pipeline/transforms/xmljoin.adoc[XML Join] * xref:pipeline/transforms/xmloutput.adoc[XML Output] +* xref:pipeline/transforms/xmloutputadvanced.adoc[XML Output (Advanced)] * xref:pipeline/transforms/xsdvalidator.adoc[XSD Validator] * xref:pipeline/transforms/xslt.adoc[XSL Transformation] * xref:pipeline/transforms/yamlinput.adoc[Yaml Input] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc index 946b46b45b..34dd775af7 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/xmloutputadvanced.adoc @@ -16,7 +16,7 @@ under the License. //// :documentationPath: /pipeline/transforms/ :language: en_US -:description: The XML Output (Advanced) transform writes rows to an XML file using a hierarchical XML tree with optional row-loop and group-by elements. +:description: The XML Output (Advanced) transform builds hierarchical XML from input rows, with optional write-to-file, XML-as-field output, splits, and XSD generation. = image:transforms/icons/AXO.svg[XML Output (Advanced) transform Icon, role="image-doc-icon"] XML Output (Advanced) @@ -25,7 +25,7 @@ under the License. | == Description -The XML Output (Advanced) transform writes rows from any source to one or more XML files, using a hierarchical, user-defined XML tree. +The XML Output (Advanced) transform builds XML from input rows using a hierarchical, user-defined tree. You can *write to file*, *append the document as a string field* (for use by a later transform), or *both*. File-oriented options apply only when a file is written. The XML tree is a recursive structure of elements, attributes and document-fragment nodes. Exactly one element in the tree must be marked as the row-*loop*: each input row produces one occurrence of that element with its full subtree. Optionally, ancestors of the loop can be marked as *group-by*: consecutive input rows that share the same group key are emitted under a single occurrence of the group element. @@ -52,7 +52,10 @@ The dialog is organized into three tabs: *File*, *Content* and *XML Tree*. |=== |Option|Description |Transform name|Name of the transform. -|Filename|Base name of the output XML file (without extension). VFS URIs are supported. +|Output|Where to send the XML: *Write to file*, *Output XML as field*, or *Write to file and output XML as field* (both). Stored in pipeline XML as codes `writetofile`, `outputvalue`, and `both`. +|XML output field|Name of the field that receives the completed XML document (one value per split when splitting is enabled). Used when *Output* is *Output XML as field* or *both*. +|Include input fields in output|When *Output* includes an XML field: if enabled (default), each emitted row contains all input fields plus the XML field; if disabled, only the XML field is emitted (narrow stream useful for chaining). +|Filename|Base name of the output XML file (without extension). VFS URIs are supported. Required when *Output* writes to a file. |Extension|File extension (without the leading dot). Defaults to `xml`. |Encoding|Character encoding for the output file. Defaults to `UTF-8`. |Include transform copy number in filename|Append the transform copy number to the filename. @@ -60,7 +63,7 @@ The dialog is organized into three tabs: *File*, *Content* and *XML Tree*. |Include time in filename|Append the system time (`HHmmss`) to the filename. |Specify custom date/time format|Use a custom date/time pattern instead of the date/time toggles above. |Date/time format|Java `SimpleDateFormat` pattern, used when the custom format toggle is on. -|Split every N rows|Maximum rows per file before rolling over to a new split. `0` = no splitting. +|Split every N rows|Maximum rows per file before rolling over to a new split, or per completed XML field segment when *Output* includes an XML field. `0` = no splitting. |Zip output file|Wrap each output file in a zip archive (one entry per file). Generated XSDs are written next to the archive, not inside it. |Do not open new file at start|Defer file creation until the first input row is received. |Do not create file if no rows|Delete the output file at the end of the run if no rows were ever written. @@ -116,8 +119,13 @@ The XML Tree tab is the visual designer for the output structure. The left pane |Loop|Marks this element as the row-loop element. Exactly one element must carry the flag. |Group-by|Marks this element as a group-by ancestor of the loop. Consecutive rows with equal `Mapped field` values share a single occurrence. |Force create|Output this node even when the value is `null` (uses the default value when set). +|Remove outer wrapper (duplicate parent tag)|For `DocumentFragment` nodes only: when the fragment's root element repeats the parent element name, strip that outer wrapper so the inner XML is inserted without a duplicated wrapper (for example when feeding XML from an upstream XML Output (Advanced) into a child fragment node). |=== +== Chaining and output-to-field + +When *Output* is *Output XML as field* or *both*, the transform adds the configured *XML output field* to the stream for each completed document (or each split). A second XML Output (Advanced) transform can map that field with a *DocumentFragment* node. Use *Remove outer wrapper* on the fragment if the inner XML already has a root tag that would duplicate the parent element in the target tree. + == Group-by behaviour For the group-by mechanism to collapse correctly, *the input rows must already be sorted by the group-by key(s)*. Use a Sort Rows transform upstream if needed. When the key changes, the open group element is closed and a new one is opened with the new key. diff --git a/integration-tests/xml/0018-xml-output-advanced-chained.hpl b/integration-tests/xml/0018-xml-output-advanced-chained.hpl new file mode 100644 index 0000000000..deee025ccc --- /dev/null +++ b/integration-tests/xml/0018-xml-output-advanced-chained.hpl @@ -0,0 +1,364 @@ + + + + + 0018-xml-output-advanced-chained + Y + Integration test: two chained XML Output (Advanced) transforms — output compared to golden files. Same data and tree as sample xml-output-advanced-chained.hpl. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 16:00:00.000 + - + 2026/05/14 16:00:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 200 + 32 + 16 + Demonstrates two XML Output (Advanced) transforms in sequence (see also xml-output-advanced-person-addresses.hpl). + 720 + + + + + people addresses grid + addresses to xml field + Y + + + addresses to xml field + people xml file and field + Y + + + + people addresses grid + DataGrid + Three people, four addresses each (long rows, sorted by person). + Y + + 1 + + none + + + + + Alice + Oak Avenue 1 + 1000 + BE + + + Alice + Elm Road 12 + 2000 + NL + + + Alice + Cedar Plaza 5 + 3000 + DE + + + Alice + Willow Court 9 + 4000 + LU + + + Bob + Pine Lane 3 + 5000 + FR + + + Bob + Maple Square 7 + 6000 + ES + + + Bob + Birch Walk 2 + 7000 + IT + + + Bob + Spruce Ring 8 + 8000 + PT + + + Carol + Aspen Row 4 + 9000 + AT + + + Carol + Hickory Path 6 + 9100 + CH + + + Carol + Linden Alley 11 + 9200 + SE + + + Carol + Redwood Drive 22 + 9300 + NO + + + + + -1 + -1 + + N + person_name + + + + String + + + -1 + -1 + + N + street + + + + String + + + -1 + -1 + + N + zip + + + + String + + + -1 + -1 + + N + country + + + + String + + + + + 80 + 192 + + + + addresses to xml field + AdvancedXMLOutput + Group-by person_name, loop address; output XML only to addressesXml; split every 4 rows = one segment per person. + Y + + 1 + + none + + + outputvalue + addressesXml + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0018-chained-segment-unused + xml + 4 + Y + N + N + N + N + Y + Y + N + + + + addresses + Element + Y + person_name + + + address + Element + Y + + + street + Element + street + + + zip + Element + zip + + + country + Element + country + + + + + + + + 320 + 192 + + + + people xml file and field + AdvancedXMLOutput + Both: final people document to temp file + peopleXml; fragment strips duplicate addresses wrapper. + Y + + 1 + + none + + + both + peopleXml + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/hop-xml-integration-output/0018-chained + xml + 0 + N + N + N + N + Y + N + Y + N + + + + people + Element + + + person + Element + Y + + + name + Element + person_name + + + addresses + Element + + + from_addresses_field + DocumentFragment + addressesXml + Y + + + + + + + + + + 560 + 192 + + + + + + diff --git a/integration-tests/xml/files/expected/0018-chained.xml b/integration-tests/xml/files/expected/0018-chained.xml new file mode 100644 index 0000000000..dd84db11b6 --- /dev/null +++ b/integration-tests/xml/files/expected/0018-chained.xml @@ -0,0 +1,21 @@ + + +Alice +

Oak Avenue 11000BE
+
Elm Road 122000NL
+
Cedar Plaza 53000DE
+
Willow Court 94000LU
+ +Bob +
Pine Lane 35000FR
+
Maple Square 76000ES
+
Birch Walk 27000IT
+
Spruce Ring 88000PT
+ +Carol +
Aspen Row 49000AT
+
Hickory Path 69100CH
+
Linden Alley 119200SE
+
Redwood Drive 229300NO
+ + diff --git a/integration-tests/xml/files/expected/0018-chained.xsd b/integration-tests/xml/files/expected/0018-chained.xsd new file mode 100644 index 0000000000..7c92f66892 --- /dev/null +++ b/integration-tests/xml/files/expected/0018-chained.xsd @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/integration-tests/xml/main-0018-xml-output-advanced-chained.hwf b/integration-tests/xml/main-0018-xml-output-advanced-chained.hwf new file mode 100644 index 0000000000..0f676945ca --- /dev/null +++ b/integration-tests/xml/main-0018-xml-output-advanced-chained.hwf @@ -0,0 +1,210 @@ + + + + main-0018-xml-output-advanced-chained + Y + Integration test: two XML Output (Advanced) transforms in sequence (addresses per person as field, then people document with DocumentFragment). + + + - + 2026/05/14 10:10:00.000 + - + 2026/05/14 10:10:00.000 + + + + + START + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 64 + 64 + + + + clean output + + DELETE_FOLDERS + + N + + + ${java.io.tmpdir}/hop-xml-integration-output + + + 10 + success_if_no_errors + N + 208 + 64 + + + + create output + + CREATE_FOLDER + + ${java.io.tmpdir}/hop-xml-integration-output + N + N + 288 + 64 + + + + 0018-xml-output-advanced-chained.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0018-xml-output-advanced-chained.hpl + Basic + + Y + + N + local + N + N + Y + N + 368 + 64 + + + + verify XML + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0018-chained.xml + ${PROJECT_HOME}/files/expected/0018-chained.xml + N + N + 544 + 64 + + + + verify XSD + + FILE_COMPARE + + ${java.io.tmpdir}/hop-xml-integration-output/0018-chained.xsd + ${PROJECT_HOME}/files/expected/0018-chained.xsd + N + N + 720 + 64 + + + + XML mismatch + + ABORT + + Generated XML does not match the expected output (${PROJECT_HOME}/files/expected/0018-chained.xml). + N + 544 + 192 + + + + XSD mismatch + + ABORT + + Generated XSD does not match the expected output (${PROJECT_HOME}/files/expected/0018-chained.xsd). + N + 720 + 192 + + + + + + START + clean output + Y + Y + Y + + + clean output + create output + Y + Y + N + + + create output + 0018-xml-output-advanced-chained.hpl + Y + Y + N + + + 0018-xml-output-advanced-chained.hpl + verify XML + Y + Y + N + + + verify XML + verify XSD + Y + Y + N + + + verify XML + XML mismatch + Y + N + N + + + verify XSD + XSD mismatch + Y + N + N + + + + + + diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java index 22d4b35f8a..ba1ad26490 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -18,9 +18,12 @@ package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; import java.io.File; +import java.io.IOException; import java.io.OutputStream; import java.io.Reader; import java.io.StringReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; @@ -39,10 +42,13 @@ import org.apache.hop.core.Const; import org.apache.hop.core.ResultFile; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.io.CountingOutputStream; import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowDataUtil; import org.apache.hop.core.util.Utils; import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.Pipeline; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransform; @@ -59,10 +65,50 @@ */ public class AdvancedXmlOutput extends BaseTransform { + private static final Class PKG = AdvancedXmlOutputMeta.class; + private static final String EOL = "\n"; private static final XMLOutputFactory XML_OUT_FACTORY = XMLOutputFactory.newInstance(); private static final XMLInputFactory XML_IN_FACTORY = createSecureInputFactory(); + /** Writes every byte to two underlying streams (e.g. file + in-memory capture). */ + private static final class TeeOutputStream extends OutputStream { + private final OutputStream a; + private final OutputStream b; + + TeeOutputStream(OutputStream a, OutputStream b) { + this.a = a; + this.b = b; + } + + @Override + public void write(int c) throws IOException { + a.write(c); + b.write(c); + } + + @Override + public void write(byte[] buf, int off, int len) throws IOException { + a.write(buf, off, len); + b.write(buf, off, len); + } + + @Override + public void flush() throws IOException { + a.flush(); + b.flush(); + } + + @Override + public void close() throws IOException { + try { + a.close(); + } finally { + b.close(); + } + } + } + private OutputStream outputStream; /** The currently-open path of {@link XmlNode}s above the loop, deepest first. */ @@ -88,6 +134,23 @@ public boolean init() { return false; } data.splitnr = 0; + data.writeToFile = meta.writesXmlFile(); + data.outputXmlField = meta.writesXmlField(); + if (data.outputXmlField && Utils.isEmpty(resolve(meta.getOutputXmlField()))) { + logError(BaseMessages.getString(PKG, "AdvancedXmlOutput.Error.MissingOutputXmlField")); + return false; + } + if (!data.writeToFile && meta.getFileSupport().isZipped()) { + logError(BaseMessages.getString(PKG, "AdvancedXmlOutput.Error.ZipRequiresFile")); + return false; + } + if (data.writeToFile && Utils.isEmpty(resolve(meta.getFileSupport().getFileName()))) { + logError(BaseMessages.getString(PKG, "AdvancedXmlOutput.Error.MissingTargetFilename")); + return false; + } + if (!data.writeToFile) { + return true; + } if (meta.getFileSupport().isDoNotOpenNewFileInit()) { return true; } @@ -119,6 +182,11 @@ public boolean processRow() throws HopException { return false; } data.inputRowMeta = getInputRowMeta(); + if (data.outputXmlField) { + data.outputRowMeta = getInputRowMeta().clone(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + data.inputRowMetaSize = getInputRowMeta().size(); + } resolveTreeStructure(); } @@ -148,10 +216,10 @@ && getLinesOutput() > 0 return false; } - // Lazy open if the user asked us to defer file creation. - if (!data.fileOpen && meta.getFileSupport().isDoNotOpenNewFileInit()) { + // Lazy open if the user asked to defer file creation, or output-to-field-only mode. + if (!data.fileOpen && (!data.writeToFile || meta.getFileSupport().isDoNotOpenNewFileInit())) { if (!openNewFile()) { - logError("Couldn't open file " + meta.getFileSupport().getFileName()); + logError(BaseMessages.getString(PKG, "AdvancedXmlOutput.Error.OpenOutputFailed")); setErrors(1); return false; } @@ -164,7 +232,16 @@ && getLinesOutput() > 0 "Error writing XML row :" + e + Const.CR + "Row: " + getInputRowMeta().getString(r), e); } - putRow(getInputRowMeta(), r); + boolean passThroughEachRow = data.writeToFile && !data.outputXmlField; + if (passThroughEachRow) { + putRow(getInputRowMeta(), r); + } else if (data.outputXmlField) { + try { + data.lastRowForXmlOutput = getInputRowMeta().cloneRow(r); + } catch (HopValueException e) { + throw new HopException(e); + } + } if (checkFeedback(getLinesOutput()) && isBasic()) { logBasic("linenr " + getLinesOutput()); @@ -297,24 +374,51 @@ private void resolveFieldIndices(XmlNode node) throws HopException { private boolean openNewFile() { data.writer = null; + if (data.outputXmlField) { + data.xmlCaptureBuffer = new java.io.ByteArrayOutputStream(); + } else { + data.xmlCaptureBuffer = null; + } try { - String physicalName = - meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, true); - String innerName = meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, false); - FileObject file = HopVfs.getFileObject(physicalName, variables); - data.currentFile = file; - data.currentFileName = physicalName; - - OutputStream fos = HopVfs.getOutputStream(file, false); - data.countingStream = new CountingOutputStream(fos); - if (meta.getFileSupport().isZipped()) { - data.zip = new ZipOutputStream(data.countingStream); - ZipEntry entry = new ZipEntry(new File(innerName).getName()); - entry.setComment("Compressed by Apache Hop"); - data.zip.putNextEntry(entry); - outputStream = data.zip; + OutputStream payloadSink; + FileObject file = null; + String physicalName = null; + + if (data.writeToFile) { + physicalName = meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, true); + String innerName = + meta.getFileSupport().buildFilename(this, getCopy(), data.splitnr, false); + file = HopVfs.getFileObject(physicalName, variables); + data.currentFile = file; + data.currentFileName = physicalName; + + OutputStream fos = HopVfs.getOutputStream(file, false); + data.countingStream = new CountingOutputStream(fos); + if (meta.getFileSupport().isZipped()) { + data.zip = new ZipOutputStream(data.countingStream); + ZipEntry entry = new ZipEntry(new File(innerName).getName()); + entry.setComment("Compressed by Apache Hop"); + data.zip.putNextEntry(entry); + payloadSink = data.zip; + } else { + data.zip = null; + payloadSink = data.countingStream; + } + } else { + data.currentFile = null; + data.currentFileName = null; + data.countingStream = null; + data.zip = null; + payloadSink = + data.xmlCaptureBuffer != null + ? data.xmlCaptureBuffer + : new java.io.ByteArrayOutputStream(); + } + + if (data.xmlCaptureBuffer != null && data.writeToFile) { + outputStream = new TeeOutputStream(data.xmlCaptureBuffer, payloadSink); } else { - outputStream = data.countingStream; + outputStream = payloadSink; } String enc = Utils.isEmpty(meta.getEncoding()) ? Const.UTF_8 : meta.getEncoding(); @@ -419,11 +523,42 @@ private void closeFile() { } finally { data.fileOpen = false; } - if (rowsWritten && meta.isGenerateXsd()) { + try { + emitOutputXmlRowIfNeeded(); + } catch (HopException e) { + logError(e.getMessage(), e); + } + if (rowsWritten && meta.isGenerateXsd() && data.writeToFile) { writeSiblingXsd(); } } + private void emitOutputXmlRowIfNeeded() throws HopException { + if (!data.outputXmlField + || !data.rowsWrittenToCurrentFile + || data.xmlCaptureBuffer == null + || data.lastRowForXmlOutput == null + || data.outputRowMeta == null) { + return; + } + String enc = Utils.isEmpty(meta.getEncoding()) ? Const.UTF_8 : meta.getEncoding(); + Charset cs; + try { + cs = Charset.forName(enc); + } catch (Exception e) { + cs = StandardCharsets.UTF_8; + } + String xml = data.xmlCaptureBuffer.toString(cs); + Object[] out; + if (meta.isIncludeInputFieldsInOutput()) { + out = RowDataUtil.addValueData(data.lastRowForXmlOutput, data.inputRowMetaSize, xml); + } else { + out = new Object[] {xml}; + } + putRow(data.outputRowMeta, out); + data.xmlCaptureBuffer = null; + } + /** * Writes a sibling .xsd schema for the current data file. Errors are logged but don't fail the * pipeline (the data file is the contract; the schema is best-effort metadata). @@ -478,6 +613,9 @@ private void discardEmptyFile() { // best effort } } + if (data.xmlCaptureBuffer != null) { + data.xmlCaptureBuffer.reset(); + } if (data.currentFile != null && data.currentFile.exists()) { // We never registered the file as a result (registration is deferred to the // first successful row write), so just delete it. @@ -539,7 +677,7 @@ private void writeRowToTree(Object[] r) throws Exception { } private void registerResultFile() { - if (!meta.getFileSupport().isAddToResultFilenames()) { + if (!data.writeToFile || !meta.getFileSupport().isAddToResultFilenames()) { return; } if (data.currentFile == null) { @@ -785,18 +923,30 @@ private void writeDocumentFragment(XmlNode node, Object[] r) throws Exception { if (Utils.isEmpty(fragment)) { return; } + fragment = stripLeadingXmlDeclaration(fragment); + if (Utils.isEmpty(fragment)) { + return; + } + boolean stripOuter = node.isStripOuterFragmentElement(); + boolean skippedOuterWrapper = false; // Wrap so we can have multiple top-level nodes. String wrapped = "" + fragment + ""; try (Reader reader = new StringReader(wrapped)) { XMLStreamReader xr = XML_IN_FACTORY.createXMLStreamReader(reader); int depth = 0; + int stripWrapperDepth = -1; while (xr.hasNext()) { int event = xr.next(); switch (event) { case XMLStreamConstants.START_ELEMENT -> { depth++; if (depth == 1 && "root".equals(xr.getLocalName())) { - break; // skip the wrapper itself + break; + } + if (stripOuter && !skippedOuterWrapper) { + skippedOuterWrapper = true; + stripWrapperDepth = depth; + break; } data.writer.writeStartElement(xr.getLocalName()); for (int i = 0; i < xr.getAttributeCount(); i++) { @@ -808,16 +958,21 @@ private void writeDocumentFragment(XmlNode node, Object[] r) throws Exception { depth--; break; } + if (stripWrapperDepth == depth) { + stripWrapperDepth = -1; + depth--; + break; + } data.writer.writeEndElement(); depth--; } case XMLStreamConstants.CHARACTERS, XMLStreamConstants.CDATA -> { - if (depth >= 2 || (depth == 1 && !"root".equals(xr.getLocalName()))) { + if (depth > 1) { data.writer.writeCharacters(xr.getText()); } } default -> { - // ignore comments, PIs, whitespace + // ignore comments, PIs, whitespace-only at root } } } @@ -835,6 +990,27 @@ private String applyTrim(String value) { return meta.isTrimValues() && value != null ? value.trim() : value; } + /** + * Removes an optional UTF-8 BOM and XML declaration so a value produced by another XML Output + * (Advanced) transform (or any generator) can be wrapped in a synthetic root for parsing. + */ + private static String stripLeadingXmlDeclaration(String fragment) { + if (fragment == null || fragment.isEmpty()) { + return fragment; + } + String s = fragment.stripLeading(); + if (!s.isEmpty() && s.charAt(0) == '\uFEFF') { + s = s.substring(1).stripLeading(); + } + if (s.startsWith(""); + if (end >= 0) { + s = s.substring(end + 2).stripLeading(); + } + } + return s; + } + private static XMLInputFactory createSecureInputFactory() { XMLInputFactory f = XMLInputFactory.newInstance(); f.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java index 4930ddd12c..81840f3921 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java @@ -17,6 +17,7 @@ package org.apache.hop.pipeline.transforms.xml.advancedxmloutput; +import java.io.ByteArrayOutputStream; import java.util.List; import java.util.zip.ZipOutputStream; import javax.xml.stream.XMLStreamWriter; @@ -73,6 +74,26 @@ public class AdvancedXmlOutputData extends BaseTransformData implements ITransfo */ public String[] currentGroupKey; + /** Last input row snapshot for attaching the XML string field (output value / both modes). */ + public Object[] lastRowForXmlOutput; + + /** True: write XML to the configured file. */ + public boolean writeToFile; + + /** + * True: one output row per completed XML document with the document in the {@code outputXmlField} + * set on {@link AdvancedXmlOutputMeta}. + */ + public boolean outputXmlField; + + /** When {@link #outputXmlField}, uncompressed XML bytes for the current file / segment. */ + public ByteArrayOutputStream xmlCaptureBuffer; + + /** Output row layout including the optional XML field. */ + public IRowMeta outputRowMeta; + + public int inputRowMetaSize; + public AdvancedXmlOutputData() { super(); this.splitnr = 0; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index daaf88f44f..c053626601 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -65,9 +65,20 @@ public class AdvancedXmlOutputDialog extends BaseTransformDialog { private static final Class PKG = AdvancedXmlOutputMeta.class; + private static final AdvancedXmlOutputMeta.XmlOutputOperation[] OPERATION_ORDER = { + AdvancedXmlOutputMeta.XmlOutputOperation.WRITE_TO_FILE, + AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE, + AdvancedXmlOutputMeta.XmlOutputOperation.BOTH + }; + private final AdvancedXmlOutputMeta input; // File tab + private Label wlOperation; + private CCombo wOperationType; + private Label wlOutputXmlField; + private TextVar wOutputXmlField; + private Button wIncludeInputFieldsInOutput; private TextVar wFilename; private Button wbFilename; private TextVar wExtension; @@ -82,6 +93,7 @@ public class AdvancedXmlOutputDialog extends BaseTransformDialog { private Button wDoNotCreateEmptyFile; private Button wAddToResult; private CCombo wEncoding; + private Button wShowFiles; // Content tab private Button wCompactFile; @@ -199,13 +211,61 @@ private void addFileTab(CTabFolder tabFolder, ModifyListener lsMod, int margin, fl.marginHeight = 3; comp.setLayout(fl); + wlOperation = new Label(comp, SWT.RIGHT); + wlOperation.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.OperationType.Label")); + PropsUi.setLook(wlOperation); + FormData fdWlOp = new FormData(); + fdWlOp.left = new FormAttachment(0, 0); + fdWlOp.right = new FormAttachment(middle, -margin); + fdWlOp.top = new FormAttachment(0, margin); + wlOperation.setLayoutData(fdWlOp); + + wOperationType = new CCombo(comp, SWT.BORDER | SWT.READ_ONLY); + PropsUi.setLook(wOperationType); + for (AdvancedXmlOutputMeta.XmlOutputOperation op : OPERATION_ORDER) { + wOperationType.add(op.getDescription()); + } + FormData fdOp = new FormData(); + fdOp.left = new FormAttachment(middle, 0); + fdOp.right = new FormAttachment(100, 0); + fdOp.top = new FormAttachment(0, margin); + wOperationType.setLayoutData(fdOp); + wOperationType.addListener( + SWT.Selection, + e -> { + updateFileWidgetsForOperation(); + lsMod.modifyText(null); + }); + + wlOutputXmlField = new Label(comp, SWT.RIGHT); + wlOutputXmlField.setText( + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.OutputXmlField.Label")); + PropsUi.setLook(wlOutputXmlField); + FormData fdWlOut = new FormData(); + fdWlOut.left = new FormAttachment(0, 0); + fdWlOut.right = new FormAttachment(middle, -margin); + fdWlOut.top = new FormAttachment(wOperationType, margin); + wlOutputXmlField.setLayoutData(fdWlOut); + + wOutputXmlField = new TextVar(variables, comp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wOutputXmlField); + wOutputXmlField.addModifyListener(lsMod); + FormData fdOutF = new FormData(); + fdOutF.left = new FormAttachment(middle, 0); + fdOutF.right = new FormAttachment(100, 0); + fdOutF.top = new FormAttachment(wOperationType, margin); + wOutputXmlField.setLayoutData(fdOutF); + + wIncludeInputFieldsInOutput = + addCheckbox(comp, wOutputXmlField, "IncludeInputFieldsInOutput", lsMod, middle, margin); + // Filename row Label lblFn = new Label(comp, SWT.RIGHT); lblFn.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.Filename.Label")); PropsUi.setLook(lblFn); FormData fdLblFn = new FormData(); fdLblFn.left = new FormAttachment(0, 0); - fdLblFn.top = new FormAttachment(0, margin); + fdLblFn.top = new FormAttachment(wIncludeInputFieldsInOutput, margin); fdLblFn.right = new FormAttachment(middle, -margin); lblFn.setLayoutData(fdLblFn); @@ -214,7 +274,7 @@ private void addFileTab(CTabFolder tabFolder, ModifyListener lsMod, int margin, wbFilename.setText(BaseMessages.getString(PKG, "System.Button.Browse")); FormData fdBtn = new FormData(); fdBtn.right = new FormAttachment(100, 0); - fdBtn.top = new FormAttachment(0, margin); + fdBtn.top = new FormAttachment(wIncludeInputFieldsInOutput, margin); wbFilename.setLayoutData(fdBtn); wFilename = new TextVar(variables, comp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); @@ -222,7 +282,7 @@ private void addFileTab(CTabFolder tabFolder, ModifyListener lsMod, int margin, wFilename.addModifyListener(lsMod); FormData fdFn = new FormData(); fdFn.left = new FormAttachment(middle, 0); - fdFn.top = new FormAttachment(0, margin); + fdFn.top = new FormAttachment(wIncludeInputFieldsInOutput, margin); fdFn.right = new FormAttachment(wbFilename, -margin); wFilename.setLayoutData(fdFn); @@ -320,7 +380,7 @@ public void widgetSelected(SelectionEvent e) { addCheckbox(comp, wDoNotOpenAtInit, "DoNotCreateEmptyFile", lsMod, middle, margin); wAddToResult = addCheckbox(comp, wDoNotCreateEmptyFile, "AddToResult", lsMod, middle, margin); - Button wShowFiles = new Button(comp, SWT.PUSH); + wShowFiles = new Button(comp, SWT.PUSH); wShowFiles.setText(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ShowFiles.Button")); PropsUi.setLook(wShowFiles); FormData fdShow = new FormData(); @@ -338,6 +398,38 @@ public void widgetSelected(SelectionEvent e) { comp.layout(); tab.setControl(comp); + updateFileWidgetsForOperation(); + } + + private void updateFileWidgetsForOperation() { + if (wOperationType == null || wOperationType.isDisposed()) { + return; + } + int idx = wOperationType.getSelectionIndex(); + if (idx < 0) { + idx = 0; + } + boolean needFile = idx == 0 || idx == 2; + boolean needField = idx == 1 || idx == 2; + wlOutputXmlField.setEnabled(needField); + wOutputXmlField.setEnabled(needField); + if (wIncludeInputFieldsInOutput != null && !wIncludeInputFieldsInOutput.isDisposed()) { + wIncludeInputFieldsInOutput.setEnabled(needField); + } + wFilename.setEnabled(needFile); + wbFilename.setEnabled(needFile); + wExtension.setEnabled(needFile); + wAddTransformnr.setEnabled(needFile); + wSpecifyFormat.setEnabled(needFile); + wSplitEvery.setEnabled(needFile || needField); + wZipped.setEnabled(needFile); + wDoNotOpenAtInit.setEnabled(needFile); + wDoNotCreateEmptyFile.setEnabled(needFile); + wAddToResult.setEnabled(needFile); + if (wShowFiles != null && !wShowFiles.isDisposed()) { + wShowFiles.setEnabled(needFile); + } + setSpecifyFormatVisibility(); } /** Pops up a dialog with up to a handful of sample filenames built from the current settings. */ @@ -604,15 +696,23 @@ private static org.eclipse.swt.graphics.Font GuiResource(Shell shell) { // --------------------------------------------------------------------------- private void setSpecifyFormatVisibility() { + int idx = + wOperationType != null && !wOperationType.isDisposed() + ? wOperationType.getSelectionIndex() + : 0; + if (idx < 0) { + idx = 0; + } + boolean needFile = idx == 0 || idx == 2; boolean specify = wSpecifyFormat != null && wSpecifyFormat.getSelection(); if (wDateTimeFormat != null) { - wDateTimeFormat.setEnabled(specify); + wDateTimeFormat.setEnabled(needFile && specify); } if (wAddDate != null) { - wAddDate.setEnabled(!specify); + wAddDate.setEnabled(needFile && !specify); } if (wAddTime != null) { - wAddTime.setEnabled(!specify); + wAddTime.setEnabled(needFile && !specify); } } @@ -707,6 +807,16 @@ public void getData() { } wFilename.setText(Const.NVL(f.getFileName(), "")); wExtension.setText(Const.NVL(f.getExtension(), "xml")); + wOperationType.select(0); + AdvancedXmlOutputMeta.XmlOutputOperation currentOp = input.getOperationType(); + for (int i = 0; i < OPERATION_ORDER.length; i++) { + if (OPERATION_ORDER[i] == currentOp) { + wOperationType.select(i); + break; + } + } + wOutputXmlField.setText(Const.NVL(input.getOutputXmlField(), "outputXml")); + wIncludeInputFieldsInOutput.setSelection(input.isIncludeInputFieldsInOutput()); wAddTransformnr.setSelection(f.isTransformNrInFilename()); wAddDate.setSelection(f.isDateInFilename()); wAddTime.setSelection(f.isTimeInFilename()); @@ -739,6 +849,7 @@ public void getData() { XmlNode root = input.getRootNode() != null ? new XmlNode(input.getRootNode()) : defaultRootNode(); wTreeDesigner.setRootNode(root); + updateFileWidgetsForOperation(); } private static XmlNode defaultRootNode() { @@ -761,6 +872,14 @@ private void ok() { } transformName = wTransformName.getText(); + int oxi = wOperationType.getSelectionIndex(); + input.setOperationType( + oxi >= 0 && oxi < OPERATION_ORDER.length + ? OPERATION_ORDER[oxi] + : AdvancedXmlOutputMeta.XmlOutputOperation.WRITE_TO_FILE); + input.setOutputXmlField(wOutputXmlField.getText()); + input.setIncludeInputFieldsInOutput(wIncludeInputFieldsInOutput.getSelection()); + XmlFileOutputSupport f = input.getFileSupport(); if (f == null) { f = new XmlFileOutputSupport(); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java index 0d50889011..d4b39d93bc 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; import org.apache.commons.vfs2.FileObject; @@ -27,12 +28,16 @@ import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.injection.Injection; import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaFactory; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransformMeta; @@ -61,10 +66,95 @@ public class AdvancedXmlOutputMeta private static final Class PKG = AdvancedXmlOutputMeta.class; + /** Write XML document(s) to the configured file. */ + public static final String OPERATION_TYPE_WRITE_TO_FILE = "writetofile"; + + /** + * Append the produced XML document as a string field (one row per completed document / split). + */ + public static final String OPERATION_TYPE_OUTPUT_VALUE = "outputvalue"; + + /** Write to the file and append the XML string field for each completed document / split. */ + public static final String OPERATION_TYPE_BOTH = "both"; + + /** + * Where to send the XML document (same pattern as JSON Output Enhanced: writetofile / outputvalue + * / both). Stored by code in pipeline XML ({@code storeWithCode}) for reliable round-trip. + */ + @Getter + public enum XmlOutputOperation implements IEnumHasCodeAndDescription { + WRITE_TO_FILE( + OPERATION_TYPE_WRITE_TO_FILE, + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.OperationType.WriteToFile")), + OUTPUT_VALUE( + OPERATION_TYPE_OUTPUT_VALUE, + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.OperationType.OutputValue")), + BOTH( + OPERATION_TYPE_BOTH, + BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.OperationType.Both")); + + private final String code; + private final String description; + + XmlOutputOperation(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return description; + } + } + /** Filename / split / zip / result options. */ @HopMetadataProperty(key = "file") private XmlFileOutputSupport fileSupport; + /** + * Destination for the transform output (same codes as {@link #OPERATION_TYPE_WRITE_TO_FILE} / + * {@link #OPERATION_TYPE_OUTPUT_VALUE} / {@link #OPERATION_TYPE_BOTH}). + */ + @Getter(AccessLevel.NONE) + @Injection(name = "", group = "GENERAL") + @HopMetadataProperty( + key = "operation_type", + storeWithCode = true, + injectionKey = "OPERATION", + injectionKeyDescription = "AdvancedXMLOutput.Injection.OPERATION") + private XmlOutputOperation operationType = XmlOutputOperation.WRITE_TO_FILE; + + /** + * Row field name for the produced XML when {@link #getOperationType()} is {@link + * XmlOutputOperation#OUTPUT_VALUE} or {@link XmlOutputOperation#BOTH}. + */ + @HopMetadataProperty(key = "output_xml_field") + private String outputXmlField; + + /** + * When true (default), each row emitted in output-to-field modes carries all input fields plus + * the generated XML. When false, only the XML field is emitted (narrow stream). + */ + @HopMetadataProperty( + key = "include_input_fields_in_output", + defaultBoolean = true, + injectionKey = "INCLUDE_INPUT_FIELDS_IN_OUTPUT", + injectionKeyDescription = "AdvancedXMLOutput.Injection.INCLUDE_INPUT_FIELDS_IN_OUTPUT") + private boolean includeInputFieldsInOutput = true; + + public XmlOutputOperation getOperationType() { + return operationType == null ? XmlOutputOperation.WRITE_TO_FILE : operationType; + } + + public void setOperationType(XmlOutputOperation operationType) { + this.operationType = operationType; + } + /** Output character encoding. */ @HopMetadataProperty( key = "encoding", @@ -181,6 +271,8 @@ public AdvancedXmlOutputMeta() { this.createEmptyElement = true; this.xslStylesheetType = "text/xsl"; this.rootNode = defaultRootNode(); + this.operationType = XmlOutputOperation.WRITE_TO_FILE; + this.outputXmlField = "outputXml"; } public AdvancedXmlOutputMeta(AdvancedXmlOutputMeta m) { @@ -202,6 +294,9 @@ public AdvancedXmlOutputMeta(AdvancedXmlOutputMeta m) { this.xslStylesheetHref = m.xslStylesheetHref; this.xslStylesheetType = m.xslStylesheetType; this.rootNode = m.rootNode == null ? defaultRootNode() : new XmlNode(m.rootNode); + this.operationType = m.getOperationType(); + this.outputXmlField = m.outputXmlField; + this.includeInputFieldsInOutput = m.isIncludeInputFieldsInOutput(); } @Override @@ -226,7 +321,37 @@ public void getFields( TransformMeta nextTransform, IVariables variables, IHopMetadataProvider metadataProvider) { - // No fields are added to the output stream; rows pass through unchanged. + if (!writesXmlField()) { + return; + } + try { + String fieldName = variables.resolve(outputXmlField); + if (!Utils.isEmpty(fieldName)) { + if (!includeInputFieldsInOutput) { + row.clear(); + } + row.addValueMeta(ValueMetaFactory.createValueMeta(fieldName, IValueMeta.TYPE_STRING)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** Resolved operation type code for metadata compatibility and samples. */ + public String resolvedOperationType() { + return getOperationType().getCode(); + } + + /** True when the pipeline should append an XML string field (output value or both). */ + public boolean writesXmlField() { + XmlOutputOperation op = getOperationType(); + return op == XmlOutputOperation.OUTPUT_VALUE || op == XmlOutputOperation.BOTH; + } + + /** True when the transform writes physical XML file(s). */ + public boolean writesXmlFile() { + XmlOutputOperation op = getOperationType(); + return op == XmlOutputOperation.WRITE_TO_FILE || op == XmlOutputOperation.BOTH; } @Override @@ -273,6 +398,33 @@ public void check( transformMeta); remarks.add(cr); } + + if (writesXmlField() && Utils.isEmpty(variables.resolve(outputXmlField))) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "AdvancedXMLOutputMeta.CheckResult.OutputXmlFieldMissing"), + transformMeta)); + } + + if (writesXmlFile() + && fileSupport != null + && Utils.isEmpty(variables.resolve(fileSupport.getFileName()))) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "AdvancedXMLOutputMeta.CheckResult.FilenameMissing"), + transformMeta)); + } + + if (!writesXmlFile() && fileSupport != null && fileSupport.isZipped()) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "AdvancedXMLOutputMeta.CheckResult.ZipNeedsFile"), + transformMeta)); + } } @Override @@ -283,7 +435,7 @@ public String exportResources( IHopMetadataProvider metadataProvider) throws HopException { try { - if (fileSupport != null && !Utils.isEmpty(fileSupport.getFileName())) { + if (writesXmlFile() && fileSupport != null && !Utils.isEmpty(fileSupport.getFileName())) { FileObject fileObject = HopVfs.getFileObject(variables.resolve(fileSupport.getFileName()), variables); fileSupport.setFileName(resourceNamingInterface.nameResource(fileObject, variables, true)); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java index a738027b02..094a26540b 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java @@ -115,6 +115,16 @@ public static NodeKind getIfPresent(String name) { @HopMetadataProperty(key = "group_by") private boolean groupBy; + /** + * When {@link #kind} is {@link NodeKind#DocumentFragment}: if true, the first element inside the + * field value is not emitted (only its contents). Use when the field already includes a wrapper + * that duplicates the parent element in the tree (e.g. field contains {@code + * ...} under a modeled {@code } element). In the tree designer + * this appears as "Remove outer wrapper (duplicate parent tag)". + */ + @HopMetadataProperty(key = "strip_outer_fragment_element") + private boolean stripOuterFragmentElement; + /** Children (only meaningful for {@link NodeKind#Element}). */ @HopMetadataProperty(key = "node", groupKey = "children", isExcludedFromInjection = true) private List children; @@ -150,6 +160,7 @@ public XmlNode(XmlNode other) { this.forceCreate = other.forceCreate; this.loop = other.loop; this.groupBy = other.groupBy; + this.stripOuterFragmentElement = other.stripOuterFragmentElement; if (other.children != null) { for (XmlNode c : other.children) { this.children.add(new XmlNode(c)); @@ -200,6 +211,7 @@ public boolean equals(Object o) { } return loop == that.loop && groupBy == that.groupBy + && stripOuterFragmentElement == that.stripOuterFragmentElement && forceCreate == that.forceCreate && type == that.type && length == that.length diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java index 51d99bf0ff..4ac5c0dd62 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java @@ -99,6 +99,7 @@ public interface ChangeListener { private Button wpLoop; private Button wpGroupBy; private Button wpForceCreate; + private Button wpStripOuterFragment; /** Root of the tree being designed; never {@code null} after {@link #setRootNode(XmlNode)}. */ private XmlNode rootNode; @@ -375,6 +376,7 @@ private void buildPropertiesPane(Composite parent) { wpLoop = newFlagButton(flagsRow, null, "Loop"); wpGroupBy = newFlagButton(flagsRow, wpLoop, "GroupBy"); wpForceCreate = newFlagButton(flagsRow, wpGroupBy, "ForceCreate"); + wpStripOuterFragment = newFlagButton(flagsRow, wpForceCreate, "StripOuterFragment"); } private Button newFlagButton(Composite parent, Button leftOf, String key) { @@ -707,6 +709,7 @@ private void populateProperties(XmlNode n) { wpLoop.setSelection(false); wpGroupBy.setSelection(false); wpForceCreate.setSelection(false); + wpStripOuterFragment.setSelection(false); return; } wpName.setText(Const.NVL(n.getName(), "")); @@ -723,6 +726,9 @@ private void populateProperties(XmlNode n) { wpLoop.setSelection(n.isLoop()); wpGroupBy.setSelection(n.isGroupBy()); wpForceCreate.setSelection(n.isForceCreate()); + wpStripOuterFragment.setSelection(n.isStripOuterFragmentElement()); + boolean frag = n.getKind() == XmlNode.NodeKind.DocumentFragment; + wpStripOuterFragment.setEnabled(frag); } finally { updatingProperties = false; } @@ -743,6 +749,7 @@ private void setPropertiesEnabled(boolean enabled) { wpLoop.setEnabled(enabled); wpGroupBy.setEnabled(enabled); wpForceCreate.setEnabled(enabled); + wpStripOuterFragment.setEnabled(enabled); } private void applyPropertiesToModel() { @@ -773,6 +780,12 @@ private void applyPropertiesToModel() { n.setLoop(wantLoop); n.setGroupBy(wpGroupBy.getSelection()); n.setForceCreate(wpForceCreate.getSelection()); + if (n.getKind() == XmlNode.NodeKind.DocumentFragment) { + n.setStripOuterFragmentElement(wpStripOuterFragment.getSelection()); + } else { + n.setStripOuterFragmentElement(false); + } + wpStripOuterFragment.setEnabled(n.getKind() == XmlNode.NodeKind.DocumentFragment); refreshSelectedTreeItemLabel(); fireChanged(); @@ -826,6 +839,9 @@ private static String formatNodeLabel(XmlNode n) { if (n.isGroupBy()) { sb.append(" [group-by]"); } + if (n.isStripOuterFragmentElement()) { + sb.append(" [strip]"); + } if (!Utils.isEmpty(n.getMappedField())) { sb.append(" ← ").append(n.getMappedField()); } else if (!Utils.isEmpty(n.getDefaultValue())) { diff --git a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties index b27782c213..703ada7fbc 100644 --- a/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties +++ b/plugins/transforms/xml/src/main/resources/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/messages/messages_en_US.properties @@ -21,6 +21,8 @@ AdvancedXMLOutput.category=Output AdvancedXmlOutputMeta.keyword=xml,output,write,tree,hierarchical,advanced AdvancedXMLOutput.Injection.FILENAME=The base name of the XML file. +AdvancedXMLOutput.Injection.OPERATION=Output destination: write to file, XML as field only, or both (codes: writetofile, outputvalue, both). +AdvancedXMLOutput.Injection.INCLUDE_INPUT_FIELDS_IN_OUTPUT=When true, output rows include all input fields plus the generated XML field; when false, only the XML field is emitted. AdvancedXMLOutput.Injection.EXTENSION=Extension appended to the file name (without leading dot). AdvancedXMLOutput.Injection.SPLIT_EVERY=Maximum rows per file before splitting (0 = no split). AdvancedXMLOutput.Injection.INC_TRANSFORMNR_IN_FILENAME=Add the transform copy number to the filename. @@ -51,6 +53,14 @@ AdvancedXMLOutput.Injection.XSL_TYPE=MIME type for the XSL stylesheet PI (defaul AdvancedXMLOutputMeta.CheckResult.TreeOk=The XML tree is structurally valid. AdvancedXMLOutputMeta.CheckResult.ExpectedInputOk=Transform is connected to a previous transform receiving rows. AdvancedXMLOutputMeta.CheckResult.ExpectedInputError=No input received from previous transforms. +AdvancedXMLOutputMeta.CheckResult.OutputXmlFieldMissing=Output mode requires the XML result field name to be set. +AdvancedXMLOutputMeta.CheckResult.FilenameMissing=Write-to-file mode requires a target filename. +AdvancedXMLOutputMeta.CheckResult.ZipNeedsFile=Zipped output requires writing to a file (not output-to-field only). + +AdvancedXmlOutput.Error.MissingOutputXmlField=Please set the output XML field name when using output to field or both. +AdvancedXmlOutput.Error.ZipRequiresFile=Zipped output is only supported when writing to a file. +AdvancedXmlOutput.Error.MissingTargetFilename=Please set the filename when writing XML to a file. +AdvancedXmlOutput.Error.OpenOutputFailed=Could not open XML output (file or in-memory buffer). # ------------------------------------------------------------ # Dialog @@ -66,6 +76,12 @@ AdvancedXMLOutputDialog.ContentTab.Title=Content AdvancedXMLOutputDialog.TreeTab.Title=XML Tree # File tab +AdvancedXMLOutputDialog.OperationType.Label=Output +AdvancedXMLOutputDialog.OperationType.WriteToFile=Write to file +AdvancedXMLOutputDialog.IncludeInputFieldsInOutput.Label=Include input fields in output +AdvancedXMLOutputDialog.OperationType.OutputValue=Output XML as field +AdvancedXMLOutputDialog.OperationType.Both=Write to file and output XML as field +AdvancedXMLOutputDialog.OutputXmlField.Label=XML output field AdvancedXMLOutputDialog.Filename.Label=Filename AdvancedXMLOutputDialog.Extension.Label=Extension AdvancedXMLOutputDialog.Encoding.Label=Encoding @@ -74,7 +90,7 @@ AdvancedXMLOutputDialog.AddDate.Label=Include date in filename AdvancedXMLOutputDialog.AddTime.Label=Include time in filename AdvancedXMLOutputDialog.SpecifyFormat.Label=Specify custom date/time format AdvancedXMLOutputDialog.DateTimeFormat.Label=Date/time format -AdvancedXMLOutputDialog.SplitEvery.Label=Split every N rows +AdvancedXMLOutputDialog.SplitEvery.Label=Split every ... rows (new file or new XML field segment) AdvancedXMLOutputDialog.Zipped.Label=Zip output file AdvancedXMLOutputDialog.DoNotOpenAtInit.Label=Do not open new file at start AdvancedXMLOutputDialog.DoNotCreateEmptyFile.Label=Do not create file if no rows @@ -136,3 +152,4 @@ AdvancedXMLOutputDialog.Properties.Grouping=Grouping AdvancedXMLOutputDialog.Properties.Loop=Loop AdvancedXMLOutputDialog.Properties.GroupBy=Group-by AdvancedXMLOutputDialog.Properties.ForceCreate=Force create +AdvancedXMLOutputDialog.Properties.StripOuterFragment=Remove outer wrapper (duplicate parent tag) diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-chained.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-chained.hpl new file mode 100644 index 0000000000..f746d185c4 --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-chained.hpl @@ -0,0 +1,364 @@ + + + + + xml-output-advanced-chained + Y + Chained XML Output (Advanced): first step builds <addresses> per person into field addressesXml (output to field + split); second step merges into <people> and writes file + peopleXml. Same data and tree as xml-output-advanced-person-addresses.hpl. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 16:00:00.000 + - + 2026/05/14 16:00:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 200 + 32 + 16 + Demonstrates two XML Output (Advanced) transforms in sequence (see also xml-output-advanced-person-addresses.hpl). + 720 + + + + + people addresses grid + addresses to xml field + Y + + + addresses to xml field + people xml file and field + Y + + + + people addresses grid + DataGrid + Three people, four addresses each (long rows, sorted by person). + Y + + 1 + + none + + + + + Alice + Oak Avenue 1 + 1000 + BE + + + Alice + Elm Road 12 + 2000 + NL + + + Alice + Cedar Plaza 5 + 3000 + DE + + + Alice + Willow Court 9 + 4000 + LU + + + Bob + Pine Lane 3 + 5000 + FR + + + Bob + Maple Square 7 + 6000 + ES + + + Bob + Birch Walk 2 + 7000 + IT + + + Bob + Spruce Ring 8 + 8000 + PT + + + Carol + Aspen Row 4 + 9000 + AT + + + Carol + Hickory Path 6 + 9100 + CH + + + Carol + Linden Alley 11 + 9200 + SE + + + Carol + Redwood Drive 22 + 9300 + NO + + + + + -1 + -1 + + N + person_name + + + + String + + + -1 + -1 + + N + street + + + + String + + + -1 + -1 + + N + zip + + + + String + + + -1 + -1 + + N + country + + + + String + + + + + 80 + 192 + + + + addresses to xml field + AdvancedXMLOutput + Group-by person_name, loop address; output XML only to addressesXml; split every 4 rows = one segment per person. + Y + + 1 + + none + + + outputvalue + addressesXml + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-chained-segment-unused + xml + 4 + Y + N + N + N + N + Y + Y + N + + + + addresses + Element + Y + person_name + + + address + Element + Y + + + street + Element + street + + + zip + Element + zip + + + country + Element + country + + + + + + + + 320 + 192 + + + + people xml file and field + AdvancedXMLOutput + Both: final people document to temp file + peopleXml; fragment strips duplicate addresses wrapper. + Y + + 1 + + none + + + both + peopleXml + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-chained + xml + 0 + N + N + N + N + Y + N + Y + N + + + + people + Element + + + person + Element + Y + + + name + Element + person_name + + + addresses + Element + + + from_addresses_field + DocumentFragment + addressesXml + Y + + + + + + + + + + 560 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-person-addresses.hpl b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-person-addresses.hpl new file mode 100644 index 0000000000..d0f4be985b --- /dev/null +++ b/plugins/transforms/xml/src/main/samples/transforms/xml-output-advanced-person-addresses.hpl @@ -0,0 +1,368 @@ + + + + + xml-output-advanced-person-addresses + Y + Long address rows: first XML Output (Advanced) builds <addresses> per person (group-by + loop) into field addressesXml using Output XML as field + Split every 4 rows; second transform merges into <people> and writes file + peopleXml. DocumentFragment removes the duplicate <addresses> wrapper. + + + Normal + + + N + 1000 + 100 + - + 2026/05/08 16:00:00.000 + - + 2026/05/14 15:00:00.000 + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Noto Sans + 10 + 200 + 32 + 16 + Grid: one row per address (4 per person × 3 people). Rows stay sorted by person so group-by works. + +First XML Output (Advanced): Output = "Output XML as field", field addressesXml, Split every 4 rows (one segment per person; must match rows-per-group). Tree: <addresses group-by person_name> <address loop> … + +Second XML Output (Advanced): Output = "Both", peopleXml + ${java.io.tmpdir}/xml-output-advanced-person-addresses.xml. Under each <person>, DocumentFragment maps addressesXml with "Remove outer wrapper (duplicate parent tag)" so the modeled <addresses> is not doubled. + 720 + + + + + people addresses grid + addresses to xml field + Y + + + addresses to xml field + people xml file and field + Y + + + + people addresses grid + DataGrid + Three people, four addresses each (long rows, sorted by person). + Y + + 1 + + none + + + + + Alice + Oak Avenue 1 + 1000 + BE + + + Alice + Elm Road 12 + 2000 + NL + + + Alice + Cedar Plaza 5 + 3000 + DE + + + Alice + Willow Court 9 + 4000 + LU + + + Bob + Pine Lane 3 + 5000 + FR + + + Bob + Maple Square 7 + 6000 + ES + + + Bob + Birch Walk 2 + 7000 + IT + + + Bob + Spruce Ring 8 + 8000 + PT + + + Carol + Aspen Row 4 + 9000 + AT + + + Carol + Hickory Path 6 + 9100 + CH + + + Carol + Linden Alley 11 + 9200 + SE + + + Carol + Redwood Drive 22 + 9300 + NO + + + + + -1 + -1 + + N + person_name + + + + String + + + -1 + -1 + + N + street + + + + String + + + -1 + -1 + + N + zip + + + + String + + + -1 + -1 + + N + country + + + + String + + + + + 80 + 192 + + + + addresses to xml field + AdvancedXMLOutput + Group-by person_name, loop address; output XML only to addressesXml; split every 4 rows = one segment per person. + Y + + 1 + + none + + + outputvalue + addressesXml + UTF-8 + N + Y + N + N + Y + N + N + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-person-addresses-segment-unused + xml + 4 + Y + N + N + N + N + Y + Y + N + + + + addresses + Element + Y + person_name + + + address + Element + Y + + + street + Element + street + + + zip + Element + zip + + + country + Element + country + + + + + + + + 320 + 192 + + + + people xml file and field + AdvancedXMLOutput + Both: final people document to temp file + peopleXml; fragment strips duplicate addresses wrapper. + Y + + 1 + + none + + + both + peopleXml + UTF-8 + N + Y + N + N + Y + N + Y + + + + + text/xsl + + ${java.io.tmpdir}/xml-output-advanced-person-addresses + xml + 0 + N + N + N + N + Y + N + Y + N + + + + people + Element + + + person + Element + Y + + + name + Element + person_name + + + addresses + Element + + + from_addresses_field + DocumentFragment + addressesXml + Y + + + + + + + + + + 560 + 192 + + + + + + diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java new file mode 100644 index 0000000000..7bbe592cbe --- /dev/null +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.xml; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.RowMetaAndData; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.HopLogStore; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.plugins.TransformPluginType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineHopMeta; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.RowProducer; +import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.pipeline.transforms.xml.advancedxmloutput.AdvancedXmlOutputMeta; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Run with {@code -Dhop.generate.chained.golden=true} to refresh {@code + * integration-tests/xml/files/expected/0018-chained.xml} and {@code .xsd} from the two Advanced XML + * Output transforms in {@code 0018-xml-output-advanced-chained.hpl} (same row set as the Data Grid + * in that pipeline). Skipped by default. + * + *

The full {@code .hpl} is not executed here because the Data Grid transform is not on this + * module's test classpath; the injector + two XML Output steps reproduce the same XML. + * + *

The copied {@code } line is normalized to single-quoted attributes so goldens match + * what Hop uses in integration tests (see {@code 0011-basic.xml}) and byte-identical {@link + * org.apache.hop.workflow.actions.filecompare.ActionFileCompare} checks succeed. + * + *

Generated XSD empty tags are normalized to self-closing form as in {@code + * 0014-document-fragment.xsd} (some StAX impls write {@code } instead of {@code />}). + */ +class XmlOutputAdvanced0018GoldenGeneratorTest { + + private static final PluginRegistry REGISTRY = PluginRegistry.getInstance(); + + @BeforeAll + static void init() throws Exception { + HopEnvironment.init(); + HopLogStore.init(); + } + + @Test + void writeExpected0018Files() throws Exception { + Assumptions.assumeTrue(Boolean.getBoolean("hop.generate.chained.golden")); + + Path moduleDir = Path.of("").toAbsolutePath(); + Path integrationHome = moduleDir.resolve("../../../integration-tests/xml").normalize(); + Path hpl = integrationHome.resolve("0018-xml-output-advanced-chained.hpl"); + Path expectedDir = integrationHome.resolve("files/expected"); + Files.createDirectories(expectedDir); + + AdvancedXmlOutputMeta meta1 = loadAdvancedXmlOutput(hpl, "addresses to xml field"); + AdvancedXmlOutputMeta meta2 = loadAdvancedXmlOutput(hpl, "people xml file and field"); + + PipelineMeta pm = new PipelineMeta(); + TransformMeta inj = PipelineTestFactory.getInjectorTransformMeta(); + TransformMeta tm1 = + new TransformMeta( + REGISTRY.getPluginId(TransformPluginType.class, meta1), + "addresses to xml field", + meta1); + tm1.setLocation(150, 50); + TransformMeta tm2 = + new TransformMeta( + REGISTRY.getPluginId(TransformPluginType.class, meta2), + "people xml file and field", + meta2); + tm2.setLocation(250, 50); + TransformMeta dum = PipelineTestFactory.getReadTransformMeta(); + dum.setLocation(350, 50); + + pm.addTransform(inj); + pm.addTransform(tm1); + pm.addTransform(tm2); + pm.addTransform(dum); + pm.addPipelineHop(new PipelineHopMeta(inj, tm1)); + pm.addPipelineHop(new PipelineHopMeta(tm1, tm2)); + pm.addPipelineHop(new PipelineHopMeta(tm2, dum)); + + Path outBase = + Path.of(System.getProperty("java.io.tmpdir")).resolve("hop-xml-integration-output"); + Files.createDirectories(outBase); + + Variables variables = new Variables(); + Pipeline pipeline = new LocalPipelineEngine(pm, variables, null); + pipeline.prepareExecution(); + + RowTransformCollector collector = new RowTransformCollector(); + pipeline.getTransform(PipelineTestFactory.DUMMY_TRANSFORMNAME, 0).addRowListener(collector); + + RowProducer rp = pipeline.addRowProducer(PipelineTestFactory.INJECTOR_TRANSFORMNAME, 0); + pipeline.startThreads(); + + List rows = peopleAddressRows(); + Iterator it = rows.iterator(); + while (it.hasNext()) { + RowMetaAndData rm = it.next(); + rp.putRow(rm.getRowMeta(), rm.getData()); + } + rp.finished(); + + pipeline.waitUntilFinished(); + Assertions.assertEquals(0L, pipeline.getResult().getNrErrors(), "pipeline must succeed"); + Assertions.assertFalse(collector.getRowsRead().isEmpty(), "dummy should receive rows"); + + Files.copy( + outBase.resolve("0018-chained.xml"), + expectedDir.resolve("0018-chained.xml"), + StandardCopyOption.REPLACE_EXISTING); + Files.copy( + outBase.resolve("0018-chained.xsd"), + expectedDir.resolve("0018-chained.xsd"), + StandardCopyOption.REPLACE_EXISTING); + normalizeHopIntegrationXmlDecl(expectedDir.resolve("0018-chained.xml")); + normalizeHopIntegrationXmlDecl(expectedDir.resolve("0018-chained.xsd")); + normalizeIntegrationXsdSelfClosingTags(expectedDir.resolve("0018-chained.xsd")); + trimTrailingContentAfterXsdSchema(expectedDir.resolve("0018-chained.xsd")); + } + + /** + * {@link AdvancedXmlOutputXsdWriter} ends the stream right after {@code } with no + * trailing newline (same as {@code 0011-basic.xsd}). + */ + private static void trimTrailingContentAfterXsdSchema(Path xsd) throws Exception { + String s = Files.readString(xsd); + String marker = ""; + int idx = s.lastIndexOf(marker); + if (idx < 0) { + return; + } + String head = s.substring(0, idx + marker.length()); + if (!s.equals(head)) { + Files.writeString(xsd, head); + } + } + + /** + * Integration {@code FILE_COMPARE} goldens use single-quoted XML declarations (e.g. {@code + * 0011-basic.xml}); some StAX configurations emit double quotes during {@code mvn test}. + */ + private static void normalizeHopIntegrationXmlDecl(Path file) throws Exception { + String s = Files.readString(file); + String n = + s.replace( + "", ""); + if (!s.equals(n)) { + Files.writeString(file, n); + } + } + + private static void normalizeIntegrationXsdSelfClosingTags(Path xsd) throws Exception { + String s = Files.readString(xsd); + String n = + Pattern.compile("") + .matcher(s) + .replaceAll(""); + n = Pattern.compile("]*)>").matcher(n).replaceAll(""); + if (!s.equals(n)) { + Files.writeString(xsd, n); + } + } + + private static AdvancedXmlOutputMeta loadAdvancedXmlOutput( + Path pipelineFile, String transformName) throws Exception { + Document doc = XmlHandler.loadXmlString(Files.readString(pipelineFile)); + NodeList transforms = doc.getElementsByTagName("transform"); + for (int i = 0; i < transforms.getLength(); i++) { + Node t = transforms.item(i); + if (!"AdvancedXMLOutput".equals(XmlHandler.getTagValue(t, "type"))) { + continue; + } + if (transformName.equals(XmlHandler.getTagValue(t, "name"))) { + return XmlMetadataUtil.deSerializeFromXml( + t, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); + } + } + throw new IllegalStateException( + "no AdvancedXMLOutput named " + transformName + " in " + pipelineFile); + } + + /** Same rows as the Data Grid in {@code 0018-xml-output-advanced-chained.hpl}. */ + private static List peopleAddressRows() throws HopException { + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("street")); + rm.addValueMeta(new ValueMetaString("zip")); + rm.addValueMeta(new ValueMetaString("country")); + List rows = new ArrayList<>(); + rows.add(new RowMetaAndData(rm, "Alice", "Oak Avenue 1", "1000", "BE")); + rows.add(new RowMetaAndData(rm, "Alice", "Elm Road 12", "2000", "NL")); + rows.add(new RowMetaAndData(rm, "Alice", "Cedar Plaza 5", "3000", "DE")); + rows.add(new RowMetaAndData(rm, "Alice", "Willow Court 9", "4000", "LU")); + rows.add(new RowMetaAndData(rm, "Bob", "Pine Lane 3", "5000", "FR")); + rows.add(new RowMetaAndData(rm, "Bob", "Maple Square 7", "6000", "ES")); + rows.add(new RowMetaAndData(rm, "Bob", "Birch Walk 2", "7000", "IT")); + rows.add(new RowMetaAndData(rm, "Bob", "Spruce Ring 8", "8000", "PT")); + rows.add(new RowMetaAndData(rm, "Carol", "Aspen Row 4", "9000", "AT")); + rows.add(new RowMetaAndData(rm, "Carol", "Hickory Path 6", "9100", "CH")); + rows.add(new RowMetaAndData(rm, "Carol", "Linden Alley 11", "9200", "SE")); + rows.add(new RowMetaAndData(rm, "Carol", "Redwood Drive 22", "9300", "NO")); + return rows; + } +} diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java index ff5b3fb8cf..e2854a04bc 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMetaTest.java @@ -22,6 +22,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -46,6 +50,9 @@ void testSerializationRoundTrip() throws Exception { assertTrue(meta.getFileSupport().isDoNotCreateEmptyFile()); assertTrue(meta.isCreateEmptyElement()); assertTrue(meta.isBlankLineAfterXmlDeclaration()); + assertTrue( + meta.isIncludeInputFieldsInOutput(), + "Legacy transform XML without include_input_fields_in_output defaults to true"); XmlNode root = meta.getRootNode(); assertNotNull(root); @@ -93,6 +100,38 @@ void testCloneCreatesIndependentTree() { assertEquals("OriginalLoop", copy.getRootNode().getChildren().get(0).getName()); } + @Test + void getFieldsOmitsInputColumnsWhenIncludeInputFieldsIsFalse() { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.setOperationType(AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE); + meta.setOutputXmlField("docXml"); + meta.setIncludeInputFieldsInOutput(false); + + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("a")); + row.addValueMeta(new ValueMetaString("b")); + meta.getFields(row, "axo", null, null, new Variables(), null); + + assertEquals(1, row.size()); + assertEquals("docXml", row.getValueMeta(0).getName()); + } + + @Test + void getFieldsKeepsInputColumnsWhenIncludeInputFieldsIsTrue() { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.setOperationType(AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE); + meta.setOutputXmlField("docXml"); + meta.setIncludeInputFieldsInOutput(true); + + IRowMeta row = new RowMeta(); + row.addValueMeta(new ValueMetaString("a")); + meta.getFields(row, "axo", null, null, new Variables(), null); + + assertEquals(2, row.size()); + assertEquals("a", row.getValueMeta(0).getName()); + assertEquals("docXml", row.getValueMeta(1).getName()); + } + @Test void testDefaultMetaValidates() { AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java index b7ba04ffaa..5bba9e82bb 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputSamplesTest.java @@ -164,6 +164,87 @@ void documentFragmentSampleEmbedsFragmentNode() throws Exception { foundFragment, "document-fragment sample must include at least one DocumentFragment node"); } + @Test + void personAddressesSampleUsesDocumentFragmentUnderAddresses() throws Exception { + AdvancedXmlOutputMeta meta = + loadAdvancedXmlOutput( + "xml-output-advanced-person-addresses.hpl", "people xml file and field"); + + XmlNode root = meta.getRootNode(); + assertEquals("people", root.getName()); + XmlNode person = root.getChildren().get(0); + assertEquals("person", person.getName()); + assertTrue(person.isLoop()); + + XmlNode addresses = + person.getChildren().stream() + .filter(c -> "addresses".equals(c.getName())) + .findFirst() + .orElseThrow(); + assertEquals(XmlNode.NodeKind.Element, addresses.getKind()); + assertEquals(1, addresses.getChildren().size()); + XmlNode frag = addresses.getChildren().get(0); + assertEquals(XmlNode.NodeKind.DocumentFragment, frag.getKind()); + assertEquals("addressesXml", frag.getMappedField()); + assertTrue( + frag.isStripOuterFragmentElement(), + "sample uses a wrapped <addresses> field; strip outer avoids double wrapper"); + + assertTrue(meta.writesXmlFile()); + assertTrue(meta.writesXmlField()); + assertEquals("both", meta.resolvedOperationType()); + assertEquals("peopleXml", meta.getOutputXmlField()); + + XmlNode name = + person.getChildren().stream() + .filter(c -> "name".equals(c.getName())) + .findFirst() + .orElseThrow(); + assertEquals("person_name", name.getMappedField()); + } + + @Test + void chainedSampleMatchesPersonAddressesTransforms() throws Exception { + AdvancedXmlOutputMeta people = + loadAdvancedXmlOutput("xml-output-advanced-chained.hpl", "people xml file and field"); + assertEquals("people", people.getRootNode().getName()); + XmlNode frag = + people.getRootNode().getChildren().get(0).getChildren().stream() + .filter(c -> "addresses".equals(c.getName())) + .findFirst() + .orElseThrow() + .getChildren() + .get(0); + assertEquals(XmlNode.NodeKind.DocumentFragment, frag.getKind()); + assertEquals("addressesXml", frag.getMappedField()); + assertTrue(frag.isStripOuterFragmentElement()); + + AdvancedXmlOutputMeta addresses = + loadAdvancedXmlOutput("xml-output-advanced-chained.hpl", "addresses to xml field"); + assertEquals( + AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE, addresses.getOperationType()); + assertEquals("addressesXml", addresses.getOutputXmlField()); + assertEquals( + "${java.io.tmpdir}/xml-output-advanced-chained-segment-unused", + addresses.getFileSupport().getFileName()); + } + + @Test + void personAddressesSampleFirstTransformOutputsAddressesSegments() throws Exception { + AdvancedXmlOutputMeta meta = + loadAdvancedXmlOutput("xml-output-advanced-person-addresses.hpl", "addresses to xml field"); + assertEquals(AdvancedXmlOutputMeta.OPERATION_TYPE_OUTPUT_VALUE, meta.resolvedOperationType()); + assertEquals(AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE, meta.getOperationType()); + assertEquals("addressesXml", meta.getOutputXmlField()); + assertEquals(4, meta.getFileSupport().getSplitEvery()); + XmlNode root = meta.getRootNode(); + assertEquals("addresses", root.getName()); + assertTrue(root.isGroupBy()); + assertEquals("person_name", root.getMappedField()); + XmlNode loop = root.getChildren().stream().filter(XmlNode::isLoop).findFirst().orElseThrow(); + assertEquals("address", loop.getName()); + } + @Test void splitSampleEnablesSplitEveryAndTransformNr() throws Exception { AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput("xml-output-advanced-split.hpl"); @@ -221,17 +302,27 @@ void compactSampleExercisesNullHandlingFlags() throws Exception { @Test void allSamplesAreWellFormedXmlAndDeserializeCleanly() throws Exception { List samples = listSamples(); - assertTrue(samples.size() >= 7, "expected the seven shipped sample pipelines"); + assertTrue(samples.size() >= 9, "expected all shipped xml-output-advanced sample pipelines"); for (Path p : samples) { Document doc = XmlHandler.loadXmlString(Files.readString(p)); assertNotNull(doc, "could not parse " + p.getFileName()); assertEquals("pipeline", doc.getDocumentElement().getNodeName(), "wrong root in " + p); - // Every shipped sample must contain exactly one AdvancedXMLOutput transform - // that round-trips through metadata serialization without errors. - AdvancedXmlOutputMeta meta = loadAdvancedXmlOutput(p.getFileName().toString()); - assertNotNull(meta.getRootNode(), "no root tree in " + p.getFileName()); - assertNotNull(findLoop(meta.getRootNode()), "no loop element in " + p.getFileName()); + NodeList transforms = doc.getElementsByTagName("transform"); + int axoCount = 0; + for (int i = 0; i < transforms.getLength(); i++) { + Node t = transforms.item(i); + if (!"AdvancedXMLOutput".equals(XmlHandler.getTagValue(t, "type"))) { + continue; + } + axoCount++; + AdvancedXmlOutputMeta meta = + XmlMetadataUtil.deSerializeFromXml( + t, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); + assertNotNull(meta.getRootNode(), "no root tree in " + p.getFileName()); + assertNotNull(findLoop(meta.getRootNode()), "no loop element in " + p.getFileName()); + } + assertTrue(axoCount > 0, "no AdvancedXMLOutput in " + p.getFileName()); } } @@ -243,19 +334,33 @@ private static AdvancedXmlOutputMeta loadAdvancedXmlOutput(String pipelineFilena throws Exception { Path path = SAMPLES_DIR.resolve(pipelineFilename); Document doc = XmlHandler.loadXmlString(Files.readString(path)); - Node transformNode = findAdvancedXmlOutputTransformNode(doc); + Node transformNode = findAdvancedXmlOutputTransformNode(doc, null); assertNotNull(transformNode, "no AdvancedXMLOutput transform found in " + pipelineFilename); return XmlMetadataUtil.deSerializeFromXml( transformNode, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); } - private static Node findAdvancedXmlOutputTransformNode(Document doc) { + private static AdvancedXmlOutputMeta loadAdvancedXmlOutput( + String pipelineFilename, String transformName) throws Exception { + Path path = SAMPLES_DIR.resolve(pipelineFilename); + Document doc = XmlHandler.loadXmlString(Files.readString(path)); + Node transformNode = findAdvancedXmlOutputTransformNode(doc, transformName); + assertNotNull(transformNode, "no AdvancedXMLOutput named " + transformName); + + return XmlMetadataUtil.deSerializeFromXml( + transformNode, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); + } + + private static Node findAdvancedXmlOutputTransformNode(Document doc, String transformName) { NodeList transforms = doc.getElementsByTagName("transform"); for (int i = 0; i < transforms.getLength(); i++) { Node t = transforms.item(i); String type = XmlHandler.getTagValue(t, "type"); - if ("AdvancedXMLOutput".equals(type)) { + if (!"AdvancedXMLOutput".equals(type)) { + continue; + } + if (transformName == null || transformName.equals(XmlHandler.getTagValue(t, "name"))) { return t; } } diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java index 6d17767709..94003fe626 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputTest.java @@ -34,11 +34,17 @@ import org.apache.hop.core.row.value.ValueMetaInteger; import org.apache.hop.core.row.value.ValueMetaNumber; import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transforms.xml.PipelineTestFactory; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; /** * End-to-end runtime tests for the XML Output (Advanced) transform. Each test runs an actual @@ -292,6 +298,235 @@ void testSplitEveryProducesMultipleFiles(@TempDir Path tempDir) throws Exception // Zipped output // --------------------------------------------------------------------------- + @Test + void testStripOuterFragmentRemovesDuplicateWrapper(@TempDir Path tempDir) throws Exception { + Path output = tempDir.resolve("frag-strip"); + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.getFileSupport().setFileName(output.toString()); + meta.getFileSupport().setExtension("xml"); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + meta.setGenerateXsd(false); + + XmlNode people = new XmlNode("people", XmlNode.NodeKind.Element); + XmlNode person = new XmlNode("person", XmlNode.NodeKind.Element); + person.setLoop(true); + XmlNode name = new XmlNode("name", XmlNode.NodeKind.Element); + name.setMappedField("person_name"); + XmlNode addresses = new XmlNode("addresses", XmlNode.NodeKind.Element); + XmlNode frag = new XmlNode("f", XmlNode.NodeKind.DocumentFragment); + frag.setMappedField("addresses_wrapped"); + frag.setStripOuterFragmentElement(true); + addresses.addChild(frag); + person.addChild(name); + person.addChild(addresses); + people.addChild(person); + meta.setRootNode(people); + + String wrapped = "

s
"; + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("addresses_wrapped")); + List rows = new ArrayList<>(); + rows.add(new RowMetaAndData(rm, "Alice", wrapped)); + + runPipeline(meta, rows); + String xml = readWrittenFile(output); + assertFalse(xml.contains(""), xml); + assertTrue(xml.contains("
"), xml); + } + + @Test + void testStripOuterWorksWhenFragmentIncludesXmlDeclaration(@TempDir Path tempDir) + throws Exception { + Path output = tempDir.resolve("frag-decl"); + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.getFileSupport().setFileName(output.toString()); + meta.getFileSupport().setExtension("xml"); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + meta.setGenerateXsd(false); + + XmlNode people = new XmlNode("people", XmlNode.NodeKind.Element); + XmlNode person = new XmlNode("person", XmlNode.NodeKind.Element); + person.setLoop(true); + XmlNode name = new XmlNode("name", XmlNode.NodeKind.Element); + name.setMappedField("person_name"); + XmlNode addresses = new XmlNode("addresses", XmlNode.NodeKind.Element); + XmlNode frag = new XmlNode("f", XmlNode.NodeKind.DocumentFragment); + frag.setMappedField("addressesXml"); + frag.setStripOuterFragmentElement(true); + addresses.addChild(frag); + person.addChild(name); + person.addChild(addresses); + people.addChild(person); + meta.setRootNode(people); + + String wrapped = + "\n" + + "
s
"; + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("addressesXml")); + List rows = new ArrayList<>(); + rows.add(new RowMetaAndData(rm, "Alice", wrapped)); + + runPipeline(meta, rows); + String xml = readWrittenFile(output); + assertFalse(xml.contains(""), xml); + assertTrue(xml.contains("
"), xml); + } + + @Test + void testOutputValueWithSplitEmitsOneRowPerSegment(@TempDir Path tempDir) throws Exception { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.setOperationType(AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE); + meta.setOutputXmlField("docXml"); + meta.getFileSupport().setSplitEvery(4); + meta.getFileSupport().setFileName(tempDir.resolve("unused").toString()); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + meta.setGenerateXsd(false); + + XmlNode root = new XmlNode("addresses", XmlNode.NodeKind.Element); + root.setGroupBy(true); + root.setMappedField("person_name"); + XmlNode address = new XmlNode("address", XmlNode.NodeKind.Element); + address.setLoop(true); + XmlNode st = new XmlNode("street", XmlNode.NodeKind.Element); + st.setMappedField("street"); + address.addChild(st); + root.addChild(address); + meta.setRootNode(root); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("street")); + List rows = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + rows.add(new RowMetaAndData(rm, "Alice", "A" + i)); + } + for (int i = 0; i < 4; i++) { + rows.add(new RowMetaAndData(rm, "Bob", "B" + i)); + } + + PipelineMeta pipelineMeta = PipelineTestFactory.generateTestTransformation(null, meta, "axo"); + List out = + PipelineTestFactory.executeTestTransformation( + pipelineMeta, + PipelineTestFactory.INJECTOR_TRANSFORMNAME, + "axo", + PipelineTestFactory.DUMMY_TRANSFORMNAME, + rows); + assertEquals(2, out.size()); + String xml0 = out.get(0).getString("docXml", ""); + String xml1 = out.get(1).getString("docXml", ""); + assertTrue(xml0.contains("A0") && xml0.contains("A3"), xml0); + assertTrue(xml1.contains("B0") && xml1.contains("B3"), xml1); + } + + @Test + void testOutputValueXmlOnlyOmitsInputFields(@TempDir Path tempDir) throws Exception { + AdvancedXmlOutputMeta meta = new AdvancedXmlOutputMeta(); + meta.setOperationType(AdvancedXmlOutputMeta.XmlOutputOperation.OUTPUT_VALUE); + meta.setOutputXmlField("docXml"); + meta.setIncludeInputFieldsInOutput(false); + meta.getFileSupport().setSplitEvery(4); + meta.getFileSupport().setFileName(tempDir.resolve("unused").toString()); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + meta.setGenerateXsd(false); + + XmlNode root = new XmlNode("addresses", XmlNode.NodeKind.Element); + root.setGroupBy(true); + root.setMappedField("person_name"); + XmlNode address = new XmlNode("address", XmlNode.NodeKind.Element); + address.setLoop(true); + XmlNode st = new XmlNode("street", XmlNode.NodeKind.Element); + st.setMappedField("street"); + address.addChild(st); + root.addChild(address); + meta.setRootNode(root); + + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("street")); + List rows = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + rows.add(new RowMetaAndData(rm, "Alice", "A" + i)); + } + for (int i = 0; i < 4; i++) { + rows.add(new RowMetaAndData(rm, "Bob", "B" + i)); + } + + PipelineMeta pipelineMeta = PipelineTestFactory.generateTestTransformation(null, meta, "axo"); + List out = + PipelineTestFactory.executeTestTransformation( + pipelineMeta, + PipelineTestFactory.INJECTOR_TRANSFORMNAME, + "axo", + PipelineTestFactory.DUMMY_TRANSFORMNAME, + rows); + assertEquals(2, out.size()); + assertEquals(1, out.get(0).getRowMeta().size()); + assertEquals("docXml", out.get(0).getRowMeta().getValueMeta(0).getName()); + String xml0 = out.get(0).getString("docXml", ""); + assertTrue(xml0.contains("A0") && xml0.contains("A3"), xml0); + } + + @Test + void deserializedPersonAddressesSampleStripsFragmentAtRuntime(@TempDir Path tempDir) + throws Exception { + AdvancedXmlOutputMeta meta = + loadMetaFromSamplePipeline( + "xml-output-advanced-person-addresses.hpl", "people xml file and field"); + XmlNode frag = + meta.getRootNode().getChildren().get(0).getChildren().stream() + .filter(c -> "addresses".equals(c.getName())) + .findFirst() + .orElseThrow() + .getChildren() + .get(0); + assertTrue( + frag.isStripOuterFragmentElement(), + "sample fragment node must deserialize strip_outer_fragment_element"); + assertEquals("addressesXml", frag.getMappedField()); + + meta.getFileSupport().setFileName(tempDir.resolve("person-addrs-from-sample").toString()); + meta.getFileSupport().setDoNotOpenNewFileInit(true); + meta.setGenerateXsd(false); + meta.setOperationType(AdvancedXmlOutputMeta.XmlOutputOperation.WRITE_TO_FILE); + + String wrapped = + "\n" + + "
x1y
"; + IRowMeta rm = new RowMeta(); + rm.addValueMeta(new ValueMetaString("person_name")); + rm.addValueMeta(new ValueMetaString("street")); + rm.addValueMeta(new ValueMetaString("zip")); + rm.addValueMeta(new ValueMetaString("country")); + rm.addValueMeta(new ValueMetaString("addressesXml")); + List rows = new ArrayList<>(); + rows.add(new RowMetaAndData(rm, "Alice", "s", "1", "BE", wrapped)); + + runPipeline(meta, rows); + String xml = readWrittenFile(tempDir.resolve("person-addrs-from-sample")); + assertFalse(xml.contains(""), xml); + } + + private static AdvancedXmlOutputMeta loadMetaFromSamplePipeline( + String filename, String transformName) throws Exception { + Path path = Path.of("src/main/samples/transforms").toAbsolutePath().resolve(filename); + Document doc = XmlHandler.loadXmlString(Files.readString(path)); + NodeList transforms = doc.getElementsByTagName("transform"); + for (int i = 0; i < transforms.getLength(); i++) { + Node t = transforms.item(i); + if ("AdvancedXMLOutput".equals(XmlHandler.getTagValue(t, "type")) + && transformName.equals(XmlHandler.getTagValue(t, "name"))) { + return XmlMetadataUtil.deSerializeFromXml( + t, AdvancedXmlOutputMeta.class, new MemoryMetadataProvider()); + } + } + throw new IllegalStateException( + "no AdvancedXMLOutput named " + transformName + " in " + filename); + } + @Test void testZippedOutputContainsValidXml(@TempDir Path tempDir) throws Exception { Path output = tempDir.resolve("zipped"); From 327b2939099a2739118248e3fc063c1b900b968d Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 14 May 2026 15:10:51 +0200 Subject: [PATCH 6/9] cleanup. fixes #7125 --- .../advancedxmloutput/AdvancedXmlOutput.java | 9 ++---- .../AdvancedXmlOutputData.java | 5 +--- .../AdvancedXmlOutputDialog.java | 8 +----- .../AdvancedXmlOutputMeta.java | 12 ++------ .../xml/advancedxmloutput/XmlNode.java | 16 +++-------- ...OutputAdvanced0018GoldenGeneratorTest.java | 28 ++++--------------- 6 files changed, 16 insertions(+), 62 deletions(-) diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java index ba1ad26490..9b385aa225 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -55,13 +55,8 @@ import org.apache.hop.pipeline.transform.TransformMeta; /** - * Runtime engine for the XML Output (Advanced) transform. - * - *

Walks the configured {@link XmlNode} tree once at first-row time, splitting it into a "prefix - * path" (root → loop's parent) with optional group-by ancestors, a "loop subtree" emitted for each - * input row, and a "suffix" of closing tags. Group-by ancestors collapse consecutive input rows - * that share the same group key into a single occurrence of the group element. Memory profile is - * O(largest group); the writer is StAX-streaming. + * Runs the XML Output (Advanced) transform: resolves the tree once, opens a StAX writer, then emits + * prefix path, per-row loop body, and closing tags (with optional group-by). */ public class AdvancedXmlOutput extends BaseTransform { diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java index 81840f3921..32b59c06d1 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputData.java @@ -80,10 +80,7 @@ public class AdvancedXmlOutputData extends BaseTransformData implements ITransfo /** True: write XML to the configured file. */ public boolean writeToFile; - /** - * True: one output row per completed XML document with the document in the {@code outputXmlField} - * set on {@link AdvancedXmlOutputMeta}. - */ + /** True when the XML document is also written to the meta's output string field. */ public boolean outputXmlField; /** When {@link #outputXmlField}, uncompressed XML bytes for the current file / segment. */ diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index c053626601..23968fe4e0 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -54,13 +54,7 @@ import org.eclipse.swt.widgets.Shell; import org.eclipse.swt.widgets.Text; -/** - * Dialog for the XML Output (Advanced) transform. - * - *

Three tabs: File (filename, encoding, splitting, zipping), Content (XML - * declaration, formatting, doctype, xsl, xsd) and XML Tree (visual designer for the - * hierarchical output structure with drag-and-drop). - */ +/** Dialog: File, Content, and XML Tree tabs. */ public class AdvancedXmlOutputDialog extends BaseTransformDialog { private static final Class PKG = AdvancedXmlOutputMeta.class; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java index d4b39d93bc..595c85476d 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -45,11 +45,7 @@ import org.apache.hop.resource.IResourceNaming; import org.apache.hop.resource.ResourceDefinition; -/** - * Metadata for the XML Output (Advanced) transform: writes input rows to one or more XML files - * following a hierarchical, user-defined XML tree (with one row-loop element and optional group-by - * ancestors). - */ +/** Metadata for the XML Output (Advanced) transform. */ @Transform( id = "AdvancedXMLOutput", image = "AXO.svg", @@ -77,10 +73,7 @@ public class AdvancedXmlOutputMeta /** Write to the file and append the XML string field for each completed document / split. */ public static final String OPERATION_TYPE_BOTH = "both"; - /** - * Where to send the XML document (same pattern as JSON Output Enhanced: writetofile / outputvalue - * / both). Stored by code in pipeline XML ({@code storeWithCode}) for reliable round-trip. - */ + /** Stored in pipeline XML by stable code, not the enum constant name. */ @Getter public enum XmlOutputOperation implements IEnumHasCodeAndDescription { WRITE_TO_FILE( @@ -368,7 +361,6 @@ public void check( CheckResult cr; - // Tree: at least one node, exactly one loop node, mapped fields exist List validationErrors = AdvancedXmlOutputValidator.validate(rootNode, prev); if (validationErrors.isEmpty()) { cr = diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java index 094a26540b..66e99f127b 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java @@ -27,13 +27,8 @@ import org.apache.hop.metadata.api.HopMetadataProperty; /** - * Describes a single node in the hierarchical XML tree of the XML Output (Advanced) transform. - * - *

An {@code XmlNode} can represent an element or an attribute. Elements may have any number of - * child nodes (recursive structure). Exactly one element in the tree must be marked as the loop - * node ({@link #loop}) to indicate where input rows are emitted. Optionally, ancestors of the loop - * node can be marked as group-by ({@link #groupBy}) to collapse consecutive rows that share the - * same group key into a single occurrence of the group element. + * One node in the transform's hierarchical XML tree: element, attribute, or document fragment. + * Exactly one element must be the row loop; ancestors of the loop may be group-by nodes. */ @Getter @Setter @@ -116,11 +111,8 @@ public static NodeKind getIfPresent(String name) { private boolean groupBy; /** - * When {@link #kind} is {@link NodeKind#DocumentFragment}: if true, the first element inside the - * field value is not emitted (only its contents). Use when the field already includes a wrapper - * that duplicates the parent element in the tree (e.g. field contains {@code - * ...} under a modeled {@code } element). In the tree designer - * this appears as "Remove outer wrapper (duplicate parent tag)". + * For {@link NodeKind#DocumentFragment}: skip the outer element in the field value when it + * duplicates the parent element in the tree. */ @HopMetadataProperty(key = "strip_outer_fragment_element") private boolean stripOuterFragmentElement; diff --git a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java index 7bbe592cbe..6b56658171 100644 --- a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java +++ b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/XmlOutputAdvanced0018GoldenGeneratorTest.java @@ -53,20 +53,10 @@ import org.w3c.dom.NodeList; /** - * Run with {@code -Dhop.generate.chained.golden=true} to refresh {@code - * integration-tests/xml/files/expected/0018-chained.xml} and {@code .xsd} from the two Advanced XML - * Output transforms in {@code 0018-xml-output-advanced-chained.hpl} (same row set as the Data Grid - * in that pipeline). Skipped by default. - * - *

The full {@code .hpl} is not executed here because the Data Grid transform is not on this - * module's test classpath; the injector + two XML Output steps reproduce the same XML. - * - *

The copied {@code } line is normalized to single-quoted attributes so goldens match - * what Hop uses in integration tests (see {@code 0011-basic.xml}) and byte-identical {@link - * org.apache.hop.workflow.actions.filecompare.ActionFileCompare} checks succeed. - * - *

Generated XSD empty tags are normalized to self-closing form as in {@code - * 0014-document-fragment.xsd} (some StAX impls write {@code } instead of {@code />}). + * Opt-in golden refresh ({@code -Dhop.generate.chained.golden=true}): runs injector + two XML + * Output (Advanced) steps with the same row set as {@code 0018-xml-output-advanced-chained.hpl}, + * then copies XML/XSD into {@code integration-tests/xml/files/expected/}. Post-processing aligns + * decl quoting, XSD self-closing tags, and trailing bytes with the existing integration goldens. */ class XmlOutputAdvanced0018GoldenGeneratorTest { @@ -156,10 +146,7 @@ void writeExpected0018Files() throws Exception { trimTrailingContentAfterXsdSchema(expectedDir.resolve("0018-chained.xsd")); } - /** - * {@link AdvancedXmlOutputXsdWriter} ends the stream right after {@code } with no - * trailing newline (same as {@code 0011-basic.xsd}). - */ + /** Drop any bytes after the closing {@code } tag. */ private static void trimTrailingContentAfterXsdSchema(Path xsd) throws Exception { String s = Files.readString(xsd); String marker = ""; @@ -173,10 +160,7 @@ private static void trimTrailingContentAfterXsdSchema(Path xsd) throws Exception } } - /** - * Integration {@code FILE_COMPARE} goldens use single-quoted XML declarations (e.g. {@code - * 0011-basic.xml}); some StAX configurations emit double quotes during {@code mvn test}. - */ + /** Match integration goldens: single-quoted {@code } declaration. */ private static void normalizeHopIntegrationXmlDecl(Path file) throws Exception { String s = Files.readString(file); String n = From 3d65140e67bf80113cb4c001c08cc32660d02d6b Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 14 May 2026 15:33:43 +0200 Subject: [PATCH 7/9] more cleanup. #7125 --- .../xml/files/expected/0011-basic.xml.bak | 7 --- .../advancedxmloutput/AdvancedXmlOutput.java | 20 --------- .../AdvancedXmlOutputMeta.java | 29 ------------ .../XmlFileOutputSupport.java | 11 ----- .../xml/advancedxmloutput/XmlNode.java | 16 ------- .../advancedxmloutput/XmlTreeDesigner.java | 44 ------------------- 6 files changed, 127 deletions(-) delete mode 100644 integration-tests/xml/files/expected/0011-basic.xml.bak diff --git a/integration-tests/xml/files/expected/0011-basic.xml.bak b/integration-tests/xml/files/expected/0011-basic.xml.bak deleted file mode 100644 index 1ceb007609..0000000000 --- a/integration-tests/xml/files/expected/0011-basic.xml.bak +++ /dev/null @@ -1,7 +0,0 @@ - - -AliceAnderson2024-01-15Y -BobBrown2024-02-03N -CarolCarter2024-04-21Y -DanielDavis2024-06-30N - diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java index 9b385aa225..1254842928 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutput.java @@ -119,10 +119,6 @@ public AdvancedXmlOutput( super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); } - // --------------------------------------------------------------------------- - // BaseTransform overrides - // --------------------------------------------------------------------------- - @Override public boolean init() { if (!super.init()) { @@ -259,10 +255,6 @@ public void dispose() { super.dispose(); } - // --------------------------------------------------------------------------- - // Tree resolution / first-row setup - // --------------------------------------------------------------------------- - /** * Walk the tree once to locate the loop node, the path from root to the loop's parent, and the * ordered list of group-by ancestors. Resolves field indices for every node that references an @@ -363,10 +355,6 @@ private void resolveFieldIndices(XmlNode node) throws HopException { } } - // --------------------------------------------------------------------------- - // File handling - // --------------------------------------------------------------------------- - private boolean openNewFile() { data.writer = null; if (data.outputXmlField) { @@ -623,10 +611,6 @@ private void discardEmptyFile() { } } - // --------------------------------------------------------------------------- - // Row-level write - // --------------------------------------------------------------------------- - private void writeRowToTree(Object[] r) throws Exception { String[] newKey = computeGroupKey(r); @@ -1013,10 +997,6 @@ private static XMLInputFactory createSecureInputFactory() { return f; } - // --------------------------------------------------------------------------- - // Helpers exposed for tests - // --------------------------------------------------------------------------- - /** Test hook: returns the current data object's writer (so unit tests can inject a mock). */ protected XMLStreamWriter getWriter() { return data == null ? null : data.writer; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java index 595c85476d..d740904cfb 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputMeta.java @@ -45,7 +45,6 @@ import org.apache.hop.resource.IResourceNaming; import org.apache.hop.resource.ResourceDefinition; -/** Metadata for the XML Output (Advanced) transform. */ @Transform( id = "AdvancedXMLOutput", image = "AXO.svg", @@ -62,18 +61,12 @@ public class AdvancedXmlOutputMeta private static final Class PKG = AdvancedXmlOutputMeta.class; - /** Write XML document(s) to the configured file. */ public static final String OPERATION_TYPE_WRITE_TO_FILE = "writetofile"; - /** - * Append the produced XML document as a string field (one row per completed document / split). - */ public static final String OPERATION_TYPE_OUTPUT_VALUE = "outputvalue"; - /** Write to the file and append the XML string field for each completed document / split. */ public static final String OPERATION_TYPE_BOTH = "both"; - /** Stored in pipeline XML by stable code, not the enum constant name. */ @Getter public enum XmlOutputOperation implements IEnumHasCodeAndDescription { WRITE_TO_FILE( @@ -105,14 +98,9 @@ public String getDescription() { } } - /** Filename / split / zip / result options. */ @HopMetadataProperty(key = "file") private XmlFileOutputSupport fileSupport; - /** - * Destination for the transform output (same codes as {@link #OPERATION_TYPE_WRITE_TO_FILE} / - * {@link #OPERATION_TYPE_OUTPUT_VALUE} / {@link #OPERATION_TYPE_BOTH}). - */ @Getter(AccessLevel.NONE) @Injection(name = "", group = "GENERAL") @HopMetadataProperty( @@ -122,10 +110,6 @@ public String getDescription() { injectionKeyDescription = "AdvancedXMLOutput.Injection.OPERATION") private XmlOutputOperation operationType = XmlOutputOperation.WRITE_TO_FILE; - /** - * Row field name for the produced XML when {@link #getOperationType()} is {@link - * XmlOutputOperation#OUTPUT_VALUE} or {@link XmlOutputOperation#BOTH}. - */ @HopMetadataProperty(key = "output_xml_field") private String outputXmlField; @@ -148,7 +132,6 @@ public void setOperationType(XmlOutputOperation operationType) { this.operationType = operationType; } - /** Output character encoding. */ @HopMetadataProperty( key = "encoding", injectionKey = "ENCODING", @@ -162,21 +145,18 @@ public void setOperationType(XmlOutputOperation operationType) { injectionKeyDescription = "AdvancedXMLOutput.Injection.COMPACT_FILE") private boolean compactFile; - /** Add a blank line after the {@code } declaration. */ @HopMetadataProperty( key = "blank_line_after_xml_decl", injectionKey = "BLANK_LINE_AFTER_XML_DECL", injectionKeyDescription = "AdvancedXMLOutput.Injection.BLANK_LINE_AFTER_XML_DECL") private boolean blankLineAfterXmlDeclaration; - /** When true, an emitted attribute keeps its tag even if the source value is null. */ @HopMetadataProperty( key = "create_attr_if_null", injectionKey = "CREATE_ATTR_IF_NULL", injectionKeyDescription = "AdvancedXMLOutput.Injection.CREATE_ATTR_IF_NULL") private boolean createAttributeIfNull; - /** When true, an attribute with no mapped field is still emitted (with empty / default value). */ @HopMetadataProperty( key = "create_attr_if_unmapped", injectionKey = "CREATE_ATTR_IF_UNMAPPED", @@ -197,28 +177,24 @@ public void setOperationType(XmlOutputOperation operationType) { injectionKeyDescription = "AdvancedXMLOutput.Injection.TRIM_VALUES") private boolean trimValues; - /** Optional global decimal separator override (per-node still wins). */ @HopMetadataProperty( key = "default_decimal_separator", injectionKey = "DEFAULT_DECIMAL_SEPARATOR", injectionKeyDescription = "AdvancedXMLOutput.Injection.DEFAULT_DECIMAL_SEPARATOR") private String defaultDecimalSeparator; - /** Optional global grouping separator override (per-node still wins). */ @HopMetadataProperty( key = "default_grouping_separator", injectionKey = "DEFAULT_GROUPING_SEPARATOR", injectionKeyDescription = "AdvancedXMLOutput.Injection.DEFAULT_GROUPING_SEPARATOR") private String defaultGroupingSeparator; - /** When true, write a sibling {@code .xsd} schema describing the produced XML structure. */ @HopMetadataProperty( key = "generate_xsd", injectionKey = "GENERATE_XSD", injectionKeyDescription = "AdvancedXMLOutput.Injection.GENERATE_XSD") private boolean generateXsd; - /** Optional DOCTYPE root element name. When set, emit DOCTYPE between the XML decl and root. */ @HopMetadataProperty( key = "doctype_root", injectionKey = "DOCTYPE_ROOT", @@ -252,7 +228,6 @@ public void setOperationType(XmlOutputOperation operationType) { injectionKeyDescription = "AdvancedXMLOutput.Injection.XSL_TYPE") private String xslStylesheetType; - /** The hierarchical XML tree definition. */ @HopMetadataProperty(key = "tree") private XmlNode rootNode; @@ -297,7 +272,6 @@ public Object clone() { return new AdvancedXmlOutputMeta(this); } - /** Default tree: {@code } so a fresh transform validates. */ private static XmlNode defaultRootNode() { XmlNode root = new XmlNode("Rows", XmlNode.NodeKind.Element); XmlNode loop = new XmlNode("Row", XmlNode.NodeKind.Element); @@ -330,18 +304,15 @@ public void getFields( } } - /** Resolved operation type code for metadata compatibility and samples. */ public String resolvedOperationType() { return getOperationType().getCode(); } - /** True when the pipeline should append an XML string field (output value or both). */ public boolean writesXmlField() { XmlOutputOperation op = getOperationType(); return op == XmlOutputOperation.OUTPUT_VALUE || op == XmlOutputOperation.BOTH; } - /** True when the transform writes physical XML file(s). */ public boolean writesXmlFile() { XmlOutputOperation op = getOperationType(); return op == XmlOutputOperation.WRITE_TO_FILE || op == XmlOutputOperation.BOTH; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java index 337a4e8255..8fa2547fec 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlFileOutputSupport.java @@ -36,77 +36,66 @@ @Setter public class XmlFileOutputSupport { - /** Base name of the output file. */ @HopMetadataProperty( key = "name", injectionKey = "FILENAME", injectionKeyDescription = "AdvancedXMLOutput.Injection.FILENAME") private String fileName; - /** Optional file extension (without leading dot). */ @HopMetadataProperty( key = "extension", injectionKey = "EXTENSION", injectionKeyDescription = "AdvancedXMLOutput.Injection.EXTENSION") private String extension; - /** Maximum number of input rows per file. 0 = unlimited (single file). */ @HopMetadataProperty( key = "splitevery", injectionKey = "SPLIT_EVERY", injectionKeyDescription = "AdvancedXMLOutput.Injection.SPLIT_EVERY") private int splitEvery; - /** Add the transform copy number to the filename. */ @HopMetadataProperty( key = "split", injectionKey = "INC_TRANSFORMNR_IN_FILENAME", injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_TRANSFORMNR_IN_FILENAME") private boolean transformNrInFilename; - /** Add the date (yyyyMMdd) to the filename. */ @HopMetadataProperty( key = "add_date", injectionKey = "INC_DATE_IN_FILENAME", injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_DATE_IN_FILENAME") private boolean dateInFilename; - /** Add the time (HHmmss) to the filename. */ @HopMetadataProperty( key = "add_time", injectionKey = "INC_TIME_IN_FILENAME", injectionKeyDescription = "AdvancedXMLOutput.Injection.INC_TIME_IN_FILENAME") private boolean timeInFilename; - /** Wrap the destination file in a zip archive. */ @HopMetadataProperty( key = "zipped", injectionKey = "ZIPPED", injectionKeyDescription = "AdvancedXMLOutput.Injection.ZIPPED") private boolean zipped; - /** Add the produced filename(s) to the pipeline result file list. */ @HopMetadataProperty( key = "add_to_result_filenames", injectionKey = "ADD_TO_RESULT", injectionKeyDescription = "AdvancedXMLOutput.Injection.ADD_TO_RESULT") private boolean addToResultFilenames; - /** Defer file creation until the first input row is received. */ @HopMetadataProperty( key = "do_not_open_newfile_init", injectionKey = "DO_NOT_CREATE_FILE_AT_STARTUP", injectionKeyDescription = "AdvancedXMLOutput.Injection.DO_NOT_CREATE_FILE_AT_STARTUP") private boolean doNotOpenNewFileInit; - /** Delete the output file at the end of the run if no rows were written. */ @HopMetadataProperty( key = "do_not_create_empty_file", injectionKey = "DO_NOT_CREATE_EMPTY_FILE", injectionKeyDescription = "AdvancedXMLOutput.Injection.DO_NOT_CREATE_EMPTY_FILE") private boolean doNotCreateEmptyFile; - /** Use a custom date-time pattern instead of the date/time flags above. */ @HopMetadataProperty( key = "SpecifyFormat", injectionKey = "SPEFICY_FORMAT", diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java index 66e99f127b..0c3c08bf06 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlNode.java @@ -36,11 +36,8 @@ public class XmlNode { @SuppressWarnings("java:S115") public enum NodeKind { - /** Standard XML element with optional text content and/or children. */ Element, - /** XML attribute on the parent element. */ Attribute, - /** Pre-built XML fragment inserted as parsed nodes (requires the source field to hold XML). */ DocumentFragment; public static NodeKind getIfPresent(String name) { @@ -48,34 +45,24 @@ public static NodeKind getIfPresent(String name) { } } - /** Local name of the element or attribute. */ @HopMetadataProperty(key = "name") private String name; - /** Optional XML namespace URI for this element (ignored for attributes in v1). */ @HopMetadataProperty(key = "namespace") private String namespace; - /** Element / Attribute / DocumentFragment. */ @HopMetadataProperty(key = "kind") private NodeKind kind; - /** - * Name of the input field whose value provides this node's content (for an element) or value (for - * an attribute / document fragment). Empty / null means "static node". - */ @HopMetadataProperty(key = "mapped_field") private String mappedField; - /** Static text used when {@link #mappedField} is empty (or the field value is null). */ @HopMetadataProperty(key = "default_value") private String defaultValue; - /** Optional Hop value-type override (one of {@link ValueMetaBase} type codes). 0 = auto. */ @HopMetadataProperty(key = "type", intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class) private int type; - /** Optional conversion mask. */ @HopMetadataProperty(key = "format") private String format; @@ -117,7 +104,6 @@ public static NodeKind getIfPresent(String name) { @HopMetadataProperty(key = "strip_outer_fragment_element") private boolean stripOuterFragmentElement; - /** Children (only meaningful for {@link NodeKind#Element}). */ @HopMetadataProperty(key = "node", groupKey = "children", isExcludedFromInjection = true) private List children; @@ -160,7 +146,6 @@ public XmlNode(XmlNode other) { } } - /** Convenience: returns true if this node has any direct children of element kind. */ public boolean hasElementChildren() { if (children == null) { return false; @@ -173,7 +158,6 @@ public boolean hasElementChildren() { return false; } - /** Convenience: returns true if this node has any direct children of attribute kind. */ public boolean hasAttributeChildren() { if (children == null) { return false; diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java index 4ac5c0dd62..cd3e10b24b 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/XmlTreeDesigner.java @@ -77,14 +77,11 @@ public interface ChangeListener { private final IVariables variables; - // Left side: input fields private org.eclipse.swt.widgets.List wInputFields; private Button wGetFields; - // Right top: tree + toolbar private Tree wTree; - // Right bottom: properties form private Text wpName; private Text wpNamespace; private CCombo wpKind; @@ -101,13 +98,10 @@ public interface ChangeListener { private Button wpForceCreate; private Button wpStripOuterFragment; - /** Root of the tree being designed; never {@code null} after {@link #setRootNode(XmlNode)}. */ private XmlNode rootNode; - /** Available input field names, used for the field combo and the drag-source list. */ private final List inputFields = new ArrayList<>(); - /** Avoid feedback loops when populating the properties form for the current selection. */ private boolean updatingProperties = false; private ChangeListener changeListener; @@ -119,10 +113,6 @@ public XmlTreeDesigner(Composite parent, int style, IVariables variables) { build(); } - // --------------------------------------------------------------------------- - // Public API used by the host dialog - // --------------------------------------------------------------------------- - public void setRootNode(XmlNode root) { this.rootNode = root == null ? defaultRoot() : root; refreshTree(); @@ -155,10 +145,6 @@ public void setGetFieldsListener(Listener listener) { } } - // --------------------------------------------------------------------------- - // UI build - // --------------------------------------------------------------------------- - private void build() { SashForm hSash = new SashForm(this, SWT.HORIZONTAL); PropsUi.setLook(hSash); @@ -430,10 +416,6 @@ private Text addLabeledText(Composite parent, Control below, String key, int lab return t; } - // --------------------------------------------------------------------------- - // DnD: dragging fields from the input list to the tree - // --------------------------------------------------------------------------- - private void setupFieldsDragSource() { Transfer[] transfers = new Transfer[] {TextTransfer.getInstance()}; DragSource source = new DragSource(wInputFields, DND.DROP_COPY | DND.DROP_MOVE); @@ -530,10 +512,6 @@ public void dropAccept(DropTargetEvent event) { }); } - // --------------------------------------------------------------------------- - // Context menu mirroring the toolbar - // --------------------------------------------------------------------------- - private void setupTreeContextMenu() { Menu menu = new Menu(wTree); @@ -586,10 +564,6 @@ private void addCheckMenu(Menu parent, String key, boolean selected, Runnable ac mi.addListener(SWT.Selection, ignore -> action.run()); } - // --------------------------------------------------------------------------- - // Tree mutations - // --------------------------------------------------------------------------- - private void addChild(XmlNode.NodeKind kind) { XmlNode parent = currentSelection(); if (parent == null) { @@ -645,12 +619,6 @@ private void moveSelected(int delta) { fireChanged(); } - /** - * Toggle the loop / group-by flag of the current selection. - * - *

Loop is mutually exclusive across the entire tree; setting one node as loop clears the flag - * elsewhere first. Group-by may co-exist on multiple ancestors of the loop. - */ private void toggleSelectedFlag(boolean toggleLoop, boolean toggleGroupBy) { XmlNode sel = currentSelection(); if (sel == null) { @@ -680,10 +648,6 @@ private void clearLoopFlag(XmlNode n) { } } - // --------------------------------------------------------------------------- - // Properties form ↔ model sync - // --------------------------------------------------------------------------- - private void handleSelectionChanged() { XmlNode n = currentSelection(); populateProperties(n); @@ -802,10 +766,6 @@ private static int parseInt(String s, int dflt) { } } - // --------------------------------------------------------------------------- - // Tree refresh / lookup - // --------------------------------------------------------------------------- - /** Rebuilds the tree from the model. Preserves expansion where possible. */ private void refreshTree() { wTree.removeAll(); @@ -909,10 +869,6 @@ private XmlNode findParentRecursive(XmlNode current, XmlNode target) { return null; } - // --------------------------------------------------------------------------- - // Misc - // --------------------------------------------------------------------------- - private void refreshInputFieldsList() { if (wInputFields == null) { return; From faab8d8902f3c25c9b7d7458063603b646925c44 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Thu, 14 May 2026 16:27:14 +0200 Subject: [PATCH 8/9] exclude golden test files for XML ITs from RAT check. #7125 --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 75cc8a1eb8..73a4c622df 100644 --- a/pom.xml +++ b/pom.xml @@ -508,6 +508,8 @@ **/integration-tests/**/*.crt **/integration-tests/**/*.csr **/integration-tests/**/mtls.conf + + **/integration-tests/xml/files/expected/** From 828f18503ddfe1d40d294f14f6a0787dd654d811 Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Mon, 18 May 2026 14:12:32 +0200 Subject: [PATCH 9/9] minor hop-web fix --- .../xml/advancedxmloutput/AdvancedXmlOutputDialog.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java index 23968fe4e0..690b63aff6 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/advancedxmloutput/AdvancedXmlOutputDialog.java @@ -736,6 +736,7 @@ private void ensureEncodingsLoaded() { // --------------------------------------------------------------------------- private void populateInputFieldsAsync() { + final Display display = shell.getDisplay(); Runnable r = () -> { TransformMeta tm = pipelineMeta.findTransform(transformName); @@ -750,7 +751,9 @@ private void populateInputFieldsAsync() { inputFieldNames.add(row.getValueMeta(i).getName()); } } - Display.getDefault().asyncExec(this::pushInputFieldsToDesigner); + if (!display.isDisposed()) { + display.asyncExec(this::pushInputFieldsToDesigner); + } } catch (HopException e) { logError(BaseMessages.getString(PKG, "AdvancedXMLOutputDialog.ErrorGettingFields"), e); }