[PDI-13713]: XML Input StAX reader doesn't add namespace info on attr…

…ibutes

There is a fix for attributes and EndElement also.
It's added unit tests.

engine/src/org/pentaho/di/trans/steps/xmlinputstream/XMLInputStream.java

@@ -29,6 +29,7 @@
 import javax.xml.stream.XMLStreamConstants;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.events.Attribute;
+import javax.xml.stream.events.EndElement;
 import javax.xml.stream.events.Namespace;
 import javax.xml.stream.events.XMLEvent;
 
@@ -254,7 +255,6 @@ private Object[] processEvent() throws KettleException {
             outputRowData[data.pos_xml_data_name] = e.asStartElement().getName().getLocalPart();
           }
         }
-
         // store the name
         if ( data.pos_xml_data_name >= 0 ) {
           data.elementName[data.elementLevel] = new String( (String) outputRowData[data.pos_xml_data_name] );
@@ -276,9 +276,7 @@ private Object[] processEvent() throws KettleException {
         break;
 
       case XMLStreamConstants.END_ELEMENT:
-        if ( data.pos_xml_data_name >= 0 ) {
-          outputRowData[data.pos_xml_data_name] = e.asEndElement().getName().getLocalPart();
-        }
+        parseEndElement( outputRowData, e.asEndElement() );
         putRowOut( outputRowData );
         data.elementParentID[data.elementLevel + 1] = null;
         data.elementLevel--;
@@ -349,6 +347,25 @@ private Object[] processEvent() throws KettleException {
     return outputRowData;
   }
 
+  private void parseEndElement( Object[] outputRowData, EndElement el ) {
+    if ( data.pos_xml_data_name >= 0 ) {
+      outputRowData[data.pos_xml_data_name] = getEndElementName( el, meta.isEnableNamespaces() );
+    }
+  }
+
+  /**Returns the qualified name of the end element
+   * @param el an EndElement event
+   * @param enabledNamespaces indicates if namespaces should be added or not 
+   * @return the qualified name of the end element
+   */
+  private String getEndElementName( EndElement el, boolean enabledNamespaces ) {
+    if ( !enabledNamespaces ) {
+      return el.getName().getLocalPart();
+    } else {
+      return getName( el.getName().getPrefix(), el.getName().getLocalPart() );
+    }
+  }
+
   // Namespaces: put an extra row out for each namespace
   @SuppressWarnings( "unchecked" )
   private Object[] parseNamespaces( Object[] outputRowData, XMLEvent e ) throws KettleValueException,
@@ -401,8 +418,7 @@ private Object[] parseAttributes( Object[] outputRowData, XMLEvent e ) throws Ke
     while ( iter.hasNext() ) {
       Object[] outputRowDataAttribute = data.outputRowMeta.cloneRow( outputRowData );
       Attribute a = iter.next();
-      outputRowDataAttribute[data.pos_xml_data_name] = a.getName().getLocalPart();
-      outputRowDataAttribute[data.pos_xml_data_value] = a.getValue();
+      parseAttribute( outputRowDataAttribute, a, meta.isEnableNamespaces() );
       if ( iter.hasNext() ) {
         // send out the Attribute row
         putRowOut( outputRowDataAttribute );
@@ -415,6 +431,36 @@ private Object[] parseAttributes( Object[] outputRowData, XMLEvent e ) throws Ke
     return outputRowData;
   }
 
+  private void parseAttribute( Object[] outputRowDataAttribute, Attribute a, boolean enabledNamespaces ) {
+    outputRowDataAttribute[data.pos_xml_data_name] = getAttributeName( a, enabledNamespaces );
+    outputRowDataAttribute[data.pos_xml_data_value] = a.getValue();
+  }
+
+  /**Returns the qualified name of the attribute
+   * @param a an attribute event
+   * @param enabledNamespaces indicates if namespaces should be added or not 
+   * @return the qualified name of the attribute
+   */
+  private String getAttributeName( Attribute a, boolean enabledNamespaces ) {
+    if ( !enabledNamespaces ) {
+      return a.getName().getLocalPart();
+    } else {
+      return getName( a.getName().getPrefix(), a.getName().getLocalPart() );
+    }
+  }
+
+  /**
+   * Returns the qualified name in the format: <code>prefix:localPart</code> if the prefix is present
+   * otherwise just <code>localPart</code>
+   * 
+   * @param prefix the namespace prefix part of the qualified name
+   * @param localPart the local part of the qualified name
+   * @return the qualified name
+   */
+  private String getName( String prefix, String localPart ) {
+    return ( !Const.isEmpty( prefix ) ) ? prefix + ":" + localPart : localPart;
+  }
+
   private void resetElementCounters() {
     data.rowNumber = new Long( 0 );
     data.elementLevel = 0;

engine/test-src/org/pentaho/di/trans/steps/xmlinputstream/XMLInputStreamTest.java

@@ -0,0 +1,292 @@
+/*! ******************************************************************************
+ *
+ * Pentaho Data Integration
+ *
+ * Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
+ *
+ *******************************************************************************
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+package org.pentaho.di.trans.steps.xmlinputstream;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.pentaho.di.core.exception.KettleException;
+import org.pentaho.di.core.exception.KettleStepException;
+import org.pentaho.di.core.logging.LoggingObjectInterface;
+import org.pentaho.di.core.row.RowMetaInterface;
+import org.pentaho.di.trans.step.RowAdapter;
+import org.pentaho.di.trans.steps.mock.StepMockHelper;
+
+/**
+ * @author Tatsiana_Kasiankova
+ * 
+ */
+public class XMLInputStreamTest {
+  private static final String INCORRECT_XML_DATA_VALUE_MESSAGE = "Incorrect xml data value - ";
+  private static final String INCORRECT_XML_DATA_NAME_MESSAGE = "Incorrect xml data name - ";
+  private static final String INCORRECT_XML_PATH_MESSAGE = "Incorrect xml path - ";
+  private static final String INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE = "Incorrect xml data type description - ";
+
+  private static final String ATTRIBUTE_2 = "ATTRIBUTE_2";
+
+  private static final String ATTRIBUTE_1 = "ATTRIBUTE_1";
+
+  private static final int START_ROW_IN_XML_TO_VERIFY = 9;
+
+  private static StepMockHelper<XMLInputStreamMeta, XMLInputStreamData> stepMockHelper;
+
+  private XMLInputStreamMeta xmlInputStreamMeta;
+
+  private XMLInputStreamData xmlInputStreamData;
+
+  private TestRowListener rl;
+
+  private int typeDescriptionPos = 0;
+  private int pathPos = 1;
+  private int dataNamePos = 2;
+  private int dataValue = 3;
+
+  @Before
+  public void setUp() throws KettleException {
+    stepMockHelper =
+        new StepMockHelper<XMLInputStreamMeta, XMLInputStreamData>( "XMLInputStreamTest", XMLInputStreamMeta.class,
+            XMLInputStreamData.class );
+    when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn(
+        stepMockHelper.logChannelInterface );
+    when( stepMockHelper.trans.isRunning() ).thenReturn( true );
+
+    xmlInputStreamMeta = new XMLInputStreamMeta();
+    xmlInputStreamMeta.setDefault();
+
+    xmlInputStreamData = new XMLInputStreamData();
+    rl = new TestRowListener();
+
+    // Turn off several options.
+    // So there are fields: xml_data_type_description - xml_path - xml_data_name - xml_data_value
+    xmlInputStreamMeta.setIncludeXmlParentPathField( false );
+    xmlInputStreamMeta.setIncludeXmlParentElementIDField( false );
+    xmlInputStreamMeta.setIncludeXmlElementIDField( false );
+    xmlInputStreamMeta.setIncludeXmlElementLevelField( false );
+  }
+
+  @After
+  public void tearDown() {
+    stepMockHelper.cleanUp();
+  }
+
+  @Test
+  public void testParseXmlWithPrefixes_WhenSetEnableNamespaceAsTrue() throws KettleException, IOException {
+    xmlInputStreamMeta.setFilename( createTestFile( getXMLString( getGroupWithPrefix() ) ) );
+    xmlInputStreamMeta.setEnableNamespaces( true );
+
+    doTest();
+
+    // Assertions
+    // check StartElement for the ProductGroup element
+    // when namespaces are enabled, we have additional NAMESPACE events - 3 for our test xml;
+    int expectedRowNum = START_ROW_IN_XML_TO_VERIFY + 3;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "START_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/Fruits:ProductGroup", rl.getWritten().get(
+        expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "Fruits:ProductGroup",
+        rl.getWritten().get( expectedRowNum )[dataNamePos] );
+
+    // attributes
+    // ATTRIBUTE_1
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/Fruits:ProductGroup", rl.getWritten().get(
+        expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "Fruits:attribute",
+        rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_1, rl.getWritten().get( expectedRowNum )[dataValue] );
+    // ATTRIBUTE_2
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/Fruits:ProductGroup", rl.getWritten().get(
+        expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "Fish:attribute", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_2, rl.getWritten().get( expectedRowNum )[dataValue] );
+
+    // check EndElement for the ProductGroup element
+    expectedRowNum = expectedRowNum + 2;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "END_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/Fruits:ProductGroup", rl.getWritten().get(
+        expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "Fruits:ProductGroup",
+        rl.getWritten().get( expectedRowNum )[dataNamePos] );
+  }
+
+  @Test
+  public void testParseXmlWithPrefixes_WhenSetEnableNamespaceAsFalse() throws KettleException, IOException {
+    xmlInputStreamMeta.setFilename( createTestFile( getXMLString( getGroupWithPrefix() ) ) );
+    xmlInputStreamMeta.setEnableNamespaces( false );
+
+    doTest();
+
+    // Assertions
+    // check StartElement for the ProductGroup element
+    int expectedRowNum = START_ROW_IN_XML_TO_VERIFY;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "START_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "ProductGroup", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+
+    // attributes
+    // ATTRIBUTE_1
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "attribute", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_1, rl.getWritten().get( expectedRowNum )[dataValue] );
+    // ATTRIBUTE_2
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "attribute", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_2, rl.getWritten().get( expectedRowNum )[dataValue] );
+
+    // check EndElement for the ProductGroup element
+    expectedRowNum = expectedRowNum + 2;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "END_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "ProductGroup", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+  }
+
+  @Test
+  public void testParseXmlWithoutPrefixes_WhenSetEnableNamespaceAsTrue() throws KettleException, IOException {
+    xmlInputStreamMeta.setFilename( createTestFile( getXMLString( getGroupWithoutPrefix() ) ) );
+    xmlInputStreamMeta.setEnableNamespaces( true );
+
+    doTest();
+
+    // Assertions
+    // check StartElement for the ProductGroup element
+    // when namespaces are enabled, we have additional NAMESPACE events - 3 for our test xml;
+    int expectedRowNum = START_ROW_IN_XML_TO_VERIFY + 3;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "START_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "ProductGroup", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+
+    // attributes
+    // ATTRIBUTE_1
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "attribute1", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_1, rl.getWritten().get( expectedRowNum )[dataValue] );
+    // ATTRIBUTE_2
+    expectedRowNum++;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "ATTRIBUTE",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "attribute2", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+    assertEquals( INCORRECT_XML_DATA_VALUE_MESSAGE, ATTRIBUTE_2, rl.getWritten().get( expectedRowNum )[dataValue] );
+
+    // check EndElement for the ProductGroup element
+    expectedRowNum = expectedRowNum + 2;
+    assertEquals( INCORRECT_XML_DATA_TYPE_DESCRIPTION_MESSAGE, "END_ELEMENT",
+        rl.getWritten().get( expectedRowNum )[typeDescriptionPos] );
+    assertEquals( INCORRECT_XML_PATH_MESSAGE, "/Products/Product/ProductGroup",
+        rl.getWritten().get( expectedRowNum )[pathPos] );
+    assertEquals( INCORRECT_XML_DATA_NAME_MESSAGE, "ProductGroup", rl.getWritten().get( expectedRowNum )[dataNamePos] );
+  }
+
+  private void doTest() throws IOException, KettleException {
+    XMLInputStream xmlInputStream =
+        new XMLInputStream( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta,
+            stepMockHelper.trans );
+
+    xmlInputStream.init( xmlInputStreamMeta, xmlInputStreamData );
+    xmlInputStream.addRowListener( rl );
+    boolean haveRowsToRead;
+    do {
+      haveRowsToRead = !xmlInputStream.processRow( xmlInputStreamMeta, xmlInputStreamData );
+
+    } while ( !haveRowsToRead );
+  }
+
+  private String createTestFile( String xmlContent ) throws IOException {
+    File tempFile = File.createTempFile( "Test", ".xml" );
+    tempFile.deleteOnExit();
+    Writer osw = new PrintWriter( tempFile, "UTF8" );
+    System.out.println( xmlContent );
+    osw.write( xmlContent );
+    osw.close();
+
+    return tempFile.getAbsolutePath();
+  }
+
+  private String getXMLString( String group ) {
+    return "<Products xmlns:Fruits=\"http://dummy.example/fruits\" xmlns:Fish=\"http://dummy.example/fish\" xmlns=\"http://dummy.example/default\">"
+        + getProduct( group ) + "</Products>";
+  }
+
+  private String getProduct( String group ) {
+    return "<Product><Id>1</Id><Name>TEST_NAME</Name>" + group + "</Product>";
+  }
+
+  private String getGroupWithPrefix() {
+    return "<Fruits:ProductGroup Fruits:attribute=\"" + ATTRIBUTE_1 + "\" Fish:attribute=\"" + ATTRIBUTE_2
+        + "\">G</Fruits:ProductGroup>";
+  }
+
+  private String getGroupWithoutPrefix() {
+    return "<ProductGroup attribute1=\"" + ATTRIBUTE_1 + "\" attribute2=\"" + ATTRIBUTE_2 + "\">G</ProductGroup>";
+  }
+
+  private class TestRowListener extends RowAdapter {
+    private List<Object[]> written = new ArrayList<Object[]>();
+
+    public List<Object[]> getWritten() {
+      return written;
+    }
+
+    @Override
+    public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
+      written.add( row );
+    }
+  }
+
+}