Skip to content

Commit

Permalink
Handle http response headers with duplicate names (e.g. cookies)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rfradinho
rfradinho committed Apr 11, 2017
1 parent a409c56 commit 3f22fc1
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 22 deletions.
20 changes: 17 additions & 3 deletions engine/src/org/pentaho/di/trans/steps/http/HTTP.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand All @@ -25,6 +25,8 @@
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;




import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Credentials;
Expand Down Expand Up @@ -177,9 +179,21 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t
encoding = contentType.replaceFirst( "^.*;\\s*charset\\s*=\\s*", "" ).replace( "\"", "" ).trim(); encoding = contentType.replaceFirst( "^.*;\\s*charset\\s*=\\s*", "" ).replace( "\"", "" ).trim();
} }
} }

JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
for ( Header header : headers ) { for ( Header header : headers ) {
json.put( header.getName(), header.getValue() ); Object previousValue = json.get( header.getName() );
if ( previousValue == null ) {
json.put( header.getName(), header.getValue() );
} else if ( previousValue instanceof List ) {
List<String> list = (List<String>) previousValue;
list.add( header.getValue() );
} else {
ArrayList<String> list = new ArrayList<String>();
list.add( (String) previousValue );
list.add( (String) header.getValue() );
json.put( header.getName(), list );
}
} }
headerString = json.toJSONString(); headerString = json.toJSONString();


Expand Down Expand Up @@ -226,7 +240,7 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t
returnFieldsOffset++; returnFieldsOffset++;
} }
if ( !Utils.isEmpty( meta.getResponseHeaderFieldName() ) ) { if ( !Utils.isEmpty( meta.getResponseHeaderFieldName() ) ) {
newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString );
} }


} finally { } finally {
Expand Down
17 changes: 15 additions & 2 deletions engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOST.java
View file
Expand Up @@ -32,6 +32,8 @@
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;


import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.Header;
Expand Down Expand Up @@ -249,7 +251,18 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException {
} }
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
for ( Header header : headers ) { for ( Header header : headers ) {
json.put( header.getName(), header.getValue() ); Object previousValue = json.get( header.getName() );
if ( previousValue == null ) {
json.put( header.getName(), header.getValue() );
} else if ( previousValue instanceof List ) {
List<String> list = (List<String>) previousValue;
list.add( header.getValue() );
} else {
ArrayList<String> list = new ArrayList<String>();
list.add( (String) previousValue );
list.add( (String) header.getValue() );
json.put( header.getName(), list );
}
} }
headerString = json.toJSONString(); headerString = json.toJSONString();


Expand Down Expand Up @@ -293,7 +306,7 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException {
returnFieldsOffset++; returnFieldsOffset++;
} }
if ( !Utils.isEmpty( meta.getResponseHeaderFieldName() ) ) { if ( !Utils.isEmpty( meta.getResponseHeaderFieldName() ) ) {
newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString );
} }
} finally { } finally {
if ( inputStreamReader != null ) { if ( inputStreamReader != null ) {
Expand Down
2 changes: 1 addition & 1 deletion engine/src/org/pentaho/di/trans/steps/rest/Rest.java
View file
Expand Up @@ -255,7 +255,7 @@ private Object[] callRest( Object[] rowData ) throws KettleException {
} }
// add response header to output // add response header to output
if ( !Utils.isEmpty( data.resultHeaderFieldName ) ) { if ( !Utils.isEmpty( data.resultHeaderFieldName ) ) {
newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString );
} }
} catch ( Exception e ) { } catch ( Exception e ) {
throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.CanNotReadURL", data.realUrl ), e ); throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.CanNotReadURL", data.realUrl ), e );
Expand Down
74 changes: 63 additions & 11 deletions engine/test-src/org/pentaho/di/trans/steps/http/HTTPIT.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand Down Expand Up @@ -36,6 +36,9 @@
import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethod;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
Expand All @@ -57,7 +60,7 @@
import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.mock.StepMockHelper; import org.pentaho.di.trans.steps.mock.StepMockHelper;


import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpServer;


Expand Down Expand Up @@ -164,7 +167,6 @@ public void setUp() throws Exception {
stepMockHelper.logChannelInterface ); stepMockHelper.logChannelInterface );
when( stepMockHelper.trans.isRunning() ).thenReturn( true ); when( stepMockHelper.trans.isRunning() ).thenReturn( true );
verify( stepMockHelper.trans, never() ).stopAll(); verify( stepMockHelper.trans, never() ).stopAll();
startHttp204Answer();
} }


@After @After
Expand All @@ -176,6 +178,7 @@ public void tearDown() throws Exception {


@Test @Test
public void test204Answer() throws Exception { public void test204Answer() throws Exception {
startHttpServer( get204AnswerHandler() );
HTTPData data = new HTTPData(); HTTPData data = new HTTPData();
int[] index = { 0, 1 }; int[] index = { 0, 1 };
RowMeta meta = new RowMeta(); RowMeta meta = new RowMeta();
Expand All @@ -200,6 +203,7 @@ public void test204Answer() throws Exception {


@Test @Test
public void testResponseHeader() throws Exception { public void testResponseHeader() throws Exception {
startHttpServer( get204AnswerHandler() );
HTTPData data = new HTTPData(); HTTPData data = new HTTPData();
int[] index = { 0, 1, 3 }; int[] index = { 0, 1, 3 };
RowMeta meta = new RowMeta(); RowMeta meta = new RowMeta();
Expand Down Expand Up @@ -231,18 +235,66 @@ public void testResponseHeader() throws Exception {


} }


private void startHttp204Answer() throws IOException {
@Test
public void testDuplicateNamesInHeader() throws Exception {
startHttpServer( getDuplicateHeadersHandler() );
HTTPData data = new HTTPData();
RowMeta meta = new RowMeta();
meta.addValueMeta( new ValueMetaString( "headerFieldName" ) );
HTTP http =
new HTTPHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false );
RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
http.setInputRowMeta( inputRowMeta );
when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
when( stepMockHelper.processRowsStepMetaInterface.getHeaderField() ).thenReturn( new String[] {} );
when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} );
when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF8" );
when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn(
"ResponseHeaderFieldName" );
http.init( stepMockHelper.processRowsStepMetaInterface, data );
Assert.assertTrue( http.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
Object[] out = ( (HTTPHandler) http ).getOutputRow();
Assert.assertTrue( out.length == 1 );
JSONParser parser = new JSONParser();
JSONObject json = (JSONObject) parser.parse( (String) out[0] );
Object userAgent = json.get( "User-agent" );
Assert.assertTrue( "HTTPTool/1.0".equals( userAgent ) );
Object cookies = json.get( "Set-cookie" );
Assert.assertTrue( cookies instanceof JSONArray );
for ( int i = 0; i < 3; i++ ) {
String cookie = ( (String) ( (JSONArray) cookies ).get( i ) );
Assert.assertTrue( cookie.startsWith( "cookie" + i ) );
}
}

private void startHttpServer( HttpHandler httpHandler ) throws IOException {
httpServer = HttpServer.create( new InetSocketAddress( HTTPIT.host, HTTPIT.port ), 10 ); httpServer = HttpServer.create( new InetSocketAddress( HTTPIT.host, HTTPIT.port ), 10 );
httpServer.createContext( "/", new HttpHandler() { httpServer.createContext( "/", httpHandler );
@Override
public void handle( HttpExchange httpExchange ) throws IOException {
httpExchange.sendResponseHeaders( 204, 0 );
httpExchange.close();
}
} );
httpServer.start(); httpServer.start();
} }


private HttpHandler get204AnswerHandler() {
return httpExchange -> {
httpExchange.sendResponseHeaders( 204, 0 );
httpExchange.close();
};
}

private HttpHandler getDuplicateHeadersHandler() {
return httpExchange -> {
Headers headers = httpExchange.getResponseHeaders();
headers.add( "User-agent", "HTTPTool/1.0" );
headers.add( "Set-cookie", "cookie0=value0; Max-Age=3600" );
headers.add( "Set-cookie", "cookie1=value1; HttpOnly" );
headers.add( "Set-cookie", "cookie2=value2; Secure" );
httpExchange.sendResponseHeaders( 200, 0 );
httpExchange.close();
};
}


// LoadSave Test is a unit test of the meta, not an integration test. Moved to new class. // LoadSave Test is a unit test of the meta, not an integration test. Moved to new class.
// MB 5/2016 // MB 5/2016
} }
58 changes: 54 additions & 4 deletions engine/test-src/org/pentaho/di/trans/steps/httppost/HTTPPOSTIT.java
View file
Expand Up @@ -37,11 +37,13 @@
import java.net.URLDecoder; import java.net.URLDecoder;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;


import com.google.common.io.ByteStreams;
import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PostMethod;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
Expand All @@ -63,6 +65,8 @@
import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.mock.StepMockHelper; import org.pentaho.di.trans.steps.mock.StepMockHelper;


import com.google.common.io.ByteStreams;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpServer;


Expand All @@ -77,7 +81,7 @@ class HTTPPOSTHandler extends HTTPPOST {
boolean override; boolean override;


public HTTPPOSTHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, public HTTPPOSTHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
Trans trans, boolean override ) { Trans trans, boolean override ) {
super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
this.override = override; this.override = override;
} }
Expand All @@ -98,7 +102,7 @@ public Object[] getRow() throws KettleException {
* to each rowset! * to each rowset!
* *
* @param row * @param row
* The row to put to the destination rowset(s). * The row to put to the destination rowset(s).
* @throws org.pentaho.di.core.exception.KettleStepException * @throws org.pentaho.di.core.exception.KettleStepException
* *
*/ */
Expand Down Expand Up @@ -162,7 +166,7 @@ public static void setupBeforeClass() throws KettleException {
public void setUp() throws Exception { public void setUp() throws Exception {
stepMockHelper = stepMockHelper =
new StepMockHelper<HTTPPOSTMeta, HTTPPOSTData>( "HTTPPOST CLIENT TEST", new StepMockHelper<HTTPPOSTMeta, HTTPPOSTData>( "HTTPPOST CLIENT TEST",
HTTPPOSTMeta.class, HTTPPOSTData.class ); HTTPPOSTMeta.class, HTTPPOSTData.class );
when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn( when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn(
stepMockHelper.logChannelInterface ); stepMockHelper.logChannelInterface );
when( stepMockHelper.trans.isRunning() ).thenReturn( true ); when( stepMockHelper.trans.isRunning() ).thenReturn( true );
Expand Down Expand Up @@ -233,6 +237,39 @@ public void testResponseHeader() throws Exception {
Assert.assertTrue( meta.equals( out, expectedRow, index ) ); Assert.assertTrue( meta.equals( out, expectedRow, index ) );
} }


@Test
public void testDuplicateNamesInHeader() throws Exception {
startHttpServer( getDuplicateHeadersHandler() );
HTTPPOSTData data = new HTTPPOSTData();
RowMeta meta = new RowMeta();
meta.addValueMeta( new ValueMetaString( "headerFieldName" ) );
HTTPPOST HTTPPOST = new HTTPPOSTHandler(
stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false );
RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
HTTPPOST.setInputRowMeta( inputRowMeta );
when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} );
when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} );
when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF-8" );
when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn(
"ResponseHeaderFieldName" );
HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data );
Assert.assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow();
Assert.assertTrue( out.length == 1 );
JSONParser parser = new JSONParser();
JSONObject json = (JSONObject) parser.parse( (String) out[0] );
Object userAgent = json.get( "User-agent" );
Assert.assertTrue( "HTTPTool/1.0".equals( userAgent ) );
Object cookies = json.get( "Set-cookie" );
Assert.assertTrue( cookies instanceof JSONArray );
for ( int i = 0; i < 3; i++ ) {
String cookie = ( (String) ( (JSONArray) cookies ).get( i ) );
Assert.assertTrue( cookie.startsWith( "cookie" + i ) );
}
}

@Test @Test
public void testUTF8() throws Exception { public void testUTF8() throws Exception {
testServerReturnsCorrectlyEncodedParams( "test string \uD842\uDFB7 øó 測試", "UTF-8" ); testServerReturnsCorrectlyEncodedParams( "test string \uD842\uDFB7 øó 測試", "UTF-8" );
Expand Down Expand Up @@ -273,6 +310,7 @@ public void testServerReturnsCorrectlyEncodedParams( String testString, String t
Assert.assertTrue( testStatus.get(), "Test failed" ); Assert.assertTrue( testStatus.get(), "Test failed" );
} }



private void startHttpServer( HttpHandler httpHandler ) throws IOException { private void startHttpServer( HttpHandler httpHandler ) throws IOException {
httpServer = HttpServer.create( new InetSocketAddress( HTTPPOSTIT.host, HTTPPOSTIT.port ), 10 ); httpServer = HttpServer.create( new InetSocketAddress( HTTPPOSTIT.host, HTTPPOSTIT.port ), 10 );
httpServer.createContext( "/", httpHandler ); httpServer.createContext( "/", httpHandler );
Expand All @@ -286,6 +324,18 @@ private HttpHandler get204AnswerHandler() {
}; };
} }


private HttpHandler getDuplicateHeadersHandler() {
return httpExchange -> {
Headers headers = httpExchange.getResponseHeaders();
headers.add( "User-agent", "HTTPTool/1.0" );
headers.add( "Set-cookie", "cookie0=value0; Max-Age=3600" );
headers.add( "Set-cookie", "cookie1=value1; HttpOnly" );
headers.add( "Set-cookie", "cookie2=value2; Secure" );
httpExchange.sendResponseHeaders( 200, 0 );
httpExchange.close();
};
}

private HttpHandler getEncodingCheckingHandler( String expectedResultString, String expectedEncoding, AtomicBoolean testStatus ) { private HttpHandler getEncodingCheckingHandler( String expectedResultString, String expectedEncoding, AtomicBoolean testStatus ) {
return httpExchange -> { return httpExchange -> {
try { try {
Expand Down
2 changes: 1 addition & 1 deletion engine/test-src/org/pentaho/di/trans/steps/rest/RestIT.java
View file
Expand Up @@ -2,7 +2,7 @@
* *
* Pentaho Data Integration * Pentaho Data Integration
* *
* Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
* *
******************************************************************************* *******************************************************************************
* *
Expand Down

0 comments on commit 3f22fc1

Please sign in to comment.