From b27396cfb538ee758117c8d917de84f17347d5dd Mon Sep 17 00:00:00 2001 From: Filipe Bojikian Rissi Date: Sun, 27 Dec 2015 01:48:36 -0200 Subject: [PATCH] [PDI-6936] - Return Response Headers in HTTP POST/REST Client Steps --- .../org/pentaho/di/trans/steps/http/HTTP.java | 55 ++++++--- .../pentaho/di/trans/steps/http/HTTPMeta.java | 21 ++++ .../http/messages/messages_en_US.properties | 3 +- .../di/trans/steps/httppost/HTTPPOST.java | 66 +++++++--- .../di/trans/steps/httppost/HTTPPOSTMeta.java | 24 +++- .../messages/messages_en_US.properties | 3 +- .../org/pentaho/di/trans/steps/rest/Rest.java | 26 ++++ .../pentaho/di/trans/steps/rest/RestData.java | 2 + .../pentaho/di/trans/steps/rest/RestMeta.java | 21 ++++ .../rest/messages/messages_en_US.properties | 3 +- .../pentaho/di/trans/steps/http/HTTPIT.java | 114 +++++++++++++++--- .../di/trans/steps/httppost/HTTPPOSTIT.java | 92 ++++++++++++-- .../pentaho/di/trans/steps/rest/RestIT.java | 80 ++++++++++-- .../di/ui/trans/steps/http/HTTPDialog.java | 25 ++++ .../trans/steps/httppost/HTTPPOSTDialog.java | 24 ++++ .../di/ui/trans/steps/rest/RestDialog.java | 24 ++++ 16 files changed, 507 insertions(+), 76 deletions(-) diff --git a/engine/src/org/pentaho/di/trans/steps/http/HTTP.java b/engine/src/org/pentaho/di/trans/steps/http/HTTP.java index e41c6f5cfde3..14c47d72e378 100644 --- a/engine/src/org/pentaho/di/trans/steps/http/HTTP.java +++ b/engine/src/org/pentaho/di/trans/steps/http/HTTP.java @@ -22,10 +22,13 @@ package org.pentaho.di.trans.steps.http; +import java.io.IOException; import java.io.InputStreamReader; import java.net.UnknownHostException; + import org.apache.commons.httpclient.Credentials; +import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; @@ -34,6 +37,7 @@ import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.util.URIUtil; +import org.json.simple.JSONObject; import org.pentaho.di.cluster.SlaveConnectionManager; import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleException; @@ -93,21 +97,21 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t // Prepare HTTP get // - HttpClient httpclient = SlaveConnectionManager.getInstance().createHttpClient(); + HttpClient httpClient = SlaveConnectionManager.getInstance().createHttpClient(); HttpMethod method = new GetMethod( url ); // Set timeout if ( data.realConnectionTimeout > -1 ) { - httpclient.getHttpConnectionManager().getParams().setConnectionTimeout( data.realConnectionTimeout ); + httpClient.getHttpConnectionManager().getParams().setConnectionTimeout( data.realConnectionTimeout ); } if ( data.realSocketTimeout > -1 ) { - httpclient.getHttpConnectionManager().getParams().setSoTimeout( data.realSocketTimeout ); + httpClient.getHttpConnectionManager().getParams().setSoTimeout( data.realSocketTimeout ); } if ( !Const.isEmpty( data.realHttpLogin ) ) { - httpclient.getParams().setAuthenticationPreemptive( true ); + httpClient.getParams().setAuthenticationPreemptive( true ); Credentials defaultcreds = new UsernamePasswordCredentials( data.realHttpLogin, data.realHttpPassword ); - httpclient.getState().setCredentials( AuthScope.ANY, defaultcreds ); + httpClient.getState().setCredentials( AuthScope.ANY, defaultcreds ); } HostConfiguration hostConfiguration = new HostConfiguration(); @@ -139,8 +143,7 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t // used for calculating the responseTime long startTime = System.currentTimeMillis(); - int statusCode = httpclient.executeMethod( hostConfiguration, method ); - + int statusCode = requestStatusCode( method, hostConfiguration, httpClient ); // calculate the responseTime long responseTime = System.currentTimeMillis() - startTime; if ( log.isDetailed() ) { @@ -148,6 +151,7 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t } String body = null; + String headerString = null; // The status code if ( isDebug() ) { logDebug( BaseMessages.getString( PKG, "HTTP.Log.ResponseStatusCode", "" + statusCode ) ); @@ -161,6 +165,7 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t if ( statusCode != 401 ) { // guess encoding // + Header[] headers = searchForHeaders( method ); String encoding = meta.getEncoding(); // Try to determine the encoding from the Content-Type value @@ -171,17 +176,18 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t encoding = contentType.replaceFirst( "^.*;\\s*charset\\s*=\\s*", "" ).replace( "\"", "" ).trim(); } } + JSONObject json = new JSONObject(); + for ( Header header : headers ) { + json.put( header.getName(), header.getValue() ); + } + headerString = json.toJSONString(); if ( isDebug() ) { log.logDebug( toString(), BaseMessages.getString( PKG, "HTTP.Log.ResponseHeaderEncoding", encoding ) ); } - // the response - if ( !Const.isEmpty( encoding ) ) { - inputStreamReader = new InputStreamReader( method.getResponseBodyAsStream(), encoding ); - } else { - inputStreamReader = new InputStreamReader( method.getResponseBodyAsStream() ); - } + inputStreamReader = openStream( encoding, method ); + StringBuilder bodyBuffer = new StringBuilder(); int c; @@ -217,6 +223,9 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t if ( !Const.isEmpty( meta.getResponseTimeFieldName() ) ) { newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( responseTime ) ); } + if ( !Const.isEmpty( meta.getResponseHeaderFieldName() ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); + } } finally { if ( inputStreamReader != null ) { @@ -225,7 +234,7 @@ private Object[] callHttpService( RowMetaInterface rowMeta, Object[] rowData ) t // Release current connection to the connection pool once you are done method.releaseConnection(); if ( data.realcloseIdleConnectionsTime > -1 ) { - httpclient.getHttpConnectionManager().closeIdleConnections( data.realcloseIdleConnectionsTime ); + httpClient.getHttpConnectionManager().closeIdleConnections( data.realcloseIdleConnectionsTime ); } } return newRow; @@ -267,6 +276,24 @@ private String determineUrl( RowMetaInterface outputRowMeta, Object[] row ) thro } } + protected int requestStatusCode( HttpMethod method, HostConfiguration hostConfiguration, HttpClient httpClient ) throws IOException { + return httpClient.executeMethod( hostConfiguration, method ); + } + + protected InputStreamReader openStream( String encoding, HttpMethod method ) throws Exception { + + if ( !Const.isEmpty( encoding ) ) { + return new InputStreamReader( method.getResponseBodyAsStream(), encoding ); + } else { + return new InputStreamReader( method.getResponseBodyAsStream() ); + } + + } + + protected Header[] searchForHeaders( HttpMethod method ) { + return method.getResponseHeaders(); + } + public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { meta = (HTTPMeta) smi; data = (HTTPData) sdi; diff --git a/engine/src/org/pentaho/di/trans/steps/http/HTTPMeta.java b/engine/src/org/pentaho/di/trans/steps/http/HTTPMeta.java index d09394f7c6e9..86793d3f7c1b 100644 --- a/engine/src/org/pentaho/di/trans/steps/http/HTTPMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/http/HTTPMeta.java @@ -33,6 +33,7 @@ import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleXMLException; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.value.ValueMetaInteger; import org.pentaho.di.core.row.value.ValueMetaString; @@ -101,6 +102,7 @@ public class HTTPMeta extends BaseStepMeta implements StepMetaInterface { private String resultCodeFieldName; private String responseTimeFieldName; + private String responseHeaderFieldName; private String[] headerParameter; private String[] headerField; @@ -324,6 +326,7 @@ public void setDefault() { fieldName = "result"; resultCodeFieldName = ""; responseTimeFieldName = ""; + responseHeaderFieldName = ""; encoding = "UTF-8"; } @@ -346,6 +349,13 @@ public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterf v.setOrigin( name ); inputRowMeta.addValueMeta( v ); } + String headerFieldName = space.environmentSubstitute( responseHeaderFieldName ); + if ( !Const.isEmpty( headerFieldName ) ) { + ValueMetaInterface v = + new ValueMeta( headerFieldName, ValueMeta.TYPE_STRING ); + v.setOrigin( name ); + inputRowMeta.addValueMeta( v ); + } } public String getXML() { @@ -385,6 +395,7 @@ public String getXML() { retval.append( " " ).append( XMLHandler.addTagValue( "name", fieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "code", resultCodeFieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "response_time", responseTimeFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "response_header", responseHeaderFieldName ) ); retval.append( " " ).append( Const.CR ); return retval.toString(); @@ -429,6 +440,7 @@ private void readData( Node stepnode, List data fieldName = XMLHandler.getTagValue( stepnode, "result", "name" ); resultCodeFieldName = XMLHandler.getTagValue( stepnode, "result", "code" ); responseTimeFieldName = XMLHandler.getTagValue( stepnode, "result", "response_time" ); + responseHeaderFieldName = XMLHandler.getTagValue( stepnode, "result", "response_header" ); } catch ( Exception e ) { throw new KettleXMLException( BaseMessages.getString( PKG, "HTTPMeta.Exception.UnableToReadStepInfo" ), e ); } @@ -466,6 +478,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis fieldName = rep.getStepAttributeString( id_step, "result_name" ); resultCodeFieldName = rep.getStepAttributeString( id_step, "result_code" ); responseTimeFieldName = rep.getStepAttributeString( id_step, "response_time" ); + responseHeaderFieldName = rep.getStepAttributeString( id_step, "response_header" ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "HTTPMeta.Exception.UnexpectedErrorReadingStepInfo" ), e ); @@ -500,6 +513,7 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform rep.saveStepAttribute( id_transformation, id_step, "result_name", fieldName ); rep.saveStepAttribute( id_transformation, id_step, "result_code", resultCodeFieldName ); rep.saveStepAttribute( id_transformation, id_step, "response_time", responseTimeFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "response_header", responseHeaderFieldName ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "HTTPMeta.Exception.UnableToSaveStepInfo" ) + id_step, e ); @@ -670,5 +684,12 @@ public String getResponseTimeFieldName() { public void setResponseTimeFieldName( String responseTimeFieldName ) { this.responseTimeFieldName = responseTimeFieldName; } + public String getResponseHeaderFieldName() { + return responseHeaderFieldName; + } + + public void setResponseHeaderFieldName( String responseHeaderFieldName ) { + this.responseHeaderFieldName = responseHeaderFieldName; + } } diff --git a/engine/src/org/pentaho/di/trans/steps/http/messages/messages_en_US.properties b/engine/src/org/pentaho/di/trans/steps/http/messages/messages_en_US.properties index f103c2fd5c98..4074e7a4dc81 100644 --- a/engine/src/org/pentaho/di/trans/steps/http/messages/messages_en_US.properties +++ b/engine/src/org/pentaho/di/trans/steps/http/messages/messages_en_US.properties @@ -28,7 +28,8 @@ HTTPDialog.URL.Label=URL HTTPDialog.GeneralTab.Title=General HTTP.Log.NoField=URL field name is missing\! HTTPDialog.Result.Label=Result field name -HTTPDialog.ResponseTime.Label=Response time (milliseconds) field name +HTTPDialog.ResponseTime.Label=Response time (milliseconds) field name +HTTPDialog.ResponseHeader.Label=Response header field name HTTPMeta.CheckResult.MissingArguments=Missing arguments, not found in input from previous steps\: HTTP.Log.ErrorFindingField=We can not find field [{0}] in the input stream\! HTTPDialog.ResultType.Label=Result type diff --git a/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOST.java b/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOST.java index 703ff02a130d..8b1f4d47ac10 100644 --- a/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOST.java +++ b/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOST.java @@ -22,13 +22,17 @@ package org.pentaho.di.trans.steps.httppost; -import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.File; import java.io.FileInputStream; +import java.io.ByteArrayInputStream; import java.io.InputStreamReader; + + import java.net.UnknownHostException; import org.apache.commons.httpclient.Credentials; +import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.NameValuePair; @@ -36,6 +40,7 @@ import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.InputStreamRequestEntity; import org.apache.commons.httpclient.methods.PostMethod; +import org.json.simple.JSONObject; import org.pentaho.di.cluster.SlaveConnectionManager; import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleException; @@ -87,22 +92,22 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { // Prepare HTTP POST // - HttpClient HTTPPOSTclient = SlaveConnectionManager.getInstance().createHttpClient(); + HttpClient httpPostClient = SlaveConnectionManager.getInstance().createHttpClient(); PostMethod post = new PostMethod( data.realUrl ); // post.setFollowRedirects(false); // Set timeout if ( data.realConnectionTimeout > -1 ) { - HTTPPOSTclient.getHttpConnectionManager().getParams().setConnectionTimeout( data.realConnectionTimeout ); + httpPostClient.getHttpConnectionManager().getParams().setConnectionTimeout( data.realConnectionTimeout ); } if ( data.realSocketTimeout > -1 ) { - HTTPPOSTclient.getHttpConnectionManager().getParams().setSoTimeout( data.realSocketTimeout ); + httpPostClient.getHttpConnectionManager().getParams().setSoTimeout( data.realSocketTimeout ); } if ( !Const.isEmpty( data.realHttpLogin ) ) { - HTTPPOSTclient.getParams().setAuthenticationPreemptive( true ); + httpPostClient.getParams().setAuthenticationPreemptive( true ); Credentials defaultcreds = new UsernamePasswordCredentials( data.realHttpLogin, data.realHttpPassword ); - HTTPPOSTclient.getState().setCredentials( AuthScope.ANY, defaultcreds ); + httpPostClient.getState().setCredentials( AuthScope.ANY, defaultcreds ); } HostConfiguration hostConfiguration = new HostConfiguration(); @@ -202,7 +207,7 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { long startTime = System.currentTimeMillis(); // Execute the POST method - int statusCode = HTTPPOSTclient.executeMethod( hostConfiguration, post ); + int statusCode = requestStatusCode( post, hostConfiguration, httpPostClient ); // calculate the responseTime long responseTime = System.currentTimeMillis() - startTime; @@ -216,6 +221,7 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { logDebug( BaseMessages.getString( PKG, "HTTPPOST.Log.ResponseCode", String.valueOf( statusCode ) ) ); } String body = null; + String headerString = null; if ( statusCode != -1 ) { if ( statusCode == 204 ) { body = ""; @@ -223,6 +229,7 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { // if the response is not 401: HTTP Authentication required if ( statusCode != 401 ) { + Header[] headers = searchForHeaders( post ); // Use request encoding if specified in component to avoid strange response encodings // See PDI-3815 String encoding = data.realEncoding; @@ -235,20 +242,15 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { encoding = contentType.replaceFirst( "^.*;\\s*charset\\s*=\\s*", "" ).replace( "\"", "" ).trim(); } } + JSONObject json = new JSONObject(); + for ( Header header : headers ) { + json.put( header.getName(), header.getValue() ); + } + headerString = json.toJSONString(); // Get the response, but only specify encoding if we've got one // otherwise the default charset ISO-8859-1 is used by HttpClient - if ( Const.isEmpty( encoding ) ) { - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "HTTPPOST.Log.Encoding", "ISO-8859-1" ) ); - } - inputStreamReader = new InputStreamReader( post.getResponseBodyAsStream() ); - } else { - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "HTTPPOST.Log.Encoding", encoding ) ); - } - inputStreamReader = new InputStreamReader( post.getResponseBodyAsStream(), encoding ); - } + inputStreamReader = openStream( encoding, post ); StringBuilder bodyBuffer = new StringBuilder(); @@ -284,6 +286,9 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { if ( !Const.isEmpty( meta.getResponseTimeFieldName() ) ) { newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( responseTime ) ); } + if ( !Const.isEmpty( meta.getResponseHeaderFieldName() ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); + } } finally { if ( inputStreamReader != null ) { inputStreamReader.close(); @@ -291,7 +296,7 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { // Release current connection to the connection pool once you are done post.releaseConnection(); if ( data.realcloseIdleConnectionsTime > -1 ) { - HTTPPOSTclient.getHttpConnectionManager().closeIdleConnections( data.realcloseIdleConnectionsTime ); + httpPostClient.getHttpConnectionManager().closeIdleConnections( data.realcloseIdleConnectionsTime ); } } return newRow; @@ -308,6 +313,29 @@ private Object[] callHTTPPOST( Object[] rowData ) throws KettleException { } } + protected int requestStatusCode( PostMethod post, HostConfiguration hostConfiguration, HttpClient httpPostClient ) throws IOException { + return httpPostClient.executeMethod( hostConfiguration, post ); + } + + protected InputStreamReader openStream( String encoding, PostMethod post ) throws Exception { + if ( Const.isEmpty( encoding ) ) { + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "HTTPPOST.Log.Encoding", "ISO-8859-1" ) ); + } + return new InputStreamReader( post.getResponseBodyAsStream() ); + } else { + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "HTTPPOST.Log.Encoding", encoding ) ); + } + return new InputStreamReader( post.getResponseBodyAsStream(), encoding ); + } + + } + + protected Header[] searchForHeaders( PostMethod post ) { + return post.getResponseHeaders(); + } + public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { meta = (HTTPPOSTMeta) smi; data = (HTTPPOSTData) sdi; diff --git a/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOSTMeta.java b/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOSTMeta.java index 0cdd422c40de..09975b72574c 100644 --- a/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOSTMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/httppost/HTTPPOSTMeta.java @@ -33,6 +33,7 @@ import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleXMLException; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.value.ValueMetaInteger; import org.pentaho.di.core.row.value.ValueMetaString; @@ -93,7 +94,7 @@ public class HTTPPOSTMeta extends BaseStepMeta implements StepMetaInterface { /** function result: new value name */ private String fieldName; private String resultCodeFieldName; - + private String responseHeaderFieldName; private boolean urlInField; private String urlField; @@ -369,6 +370,7 @@ public void setDefault() { fieldName = "result"; resultCodeFieldName = ""; responseTimeFieldName = ""; + responseHeaderFieldName = ""; postafile = false; socketTimeout = String.valueOf( DEFAULT_SOCKET_TIMEOUT ); @@ -393,6 +395,13 @@ public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterf new ValueMetaInteger( space.environmentSubstitute( responseTimeFieldName ) ); inputRowMeta.addValueMeta( v ); } + String headerFieldName = space.environmentSubstitute( responseHeaderFieldName ); + if ( !Const.isEmpty( headerFieldName ) ) { + ValueMetaInterface v = + new ValueMeta( headerFieldName, ValueMeta.TYPE_STRING ); + v.setOrigin( name ); + inputRowMeta.addValueMeta( v ); + } } public String getXML() { @@ -435,6 +444,7 @@ public String getXML() { retval.append( " " + XMLHandler.addTagValue( "name", fieldName ) ); retval.append( " " + XMLHandler.addTagValue( "code", resultCodeFieldName ) ); retval.append( " " + XMLHandler.addTagValue( "response_time", responseTimeFieldName ) ); + retval.append( " " + XMLHandler.addTagValue( "response_header", responseHeaderFieldName ) ); retval.append( " " + Const.CR ); return retval.toString(); @@ -480,6 +490,8 @@ private void readData( Node stepnode, List data fieldName = XMLHandler.getTagValue( stepnode, "result", "name" ); // Optional, can be null resultCodeFieldName = XMLHandler.getTagValue( stepnode, "result", "code" ); // Optional, can be null responseTimeFieldName = XMLHandler.getTagValue( stepnode, "result", "response_time" ); // Optional, can be null + responseHeaderFieldName = + XMLHandler.getTagValue( stepnode, "result", "response_header" ); // Optional, can be null } catch ( Exception e ) { throw new KettleXMLException( BaseMessages.getString( PKG, "HTTPPOSTMeta.Exception.UnableToReadStepInfo" ), e ); @@ -523,6 +535,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis fieldName = rep.getStepAttributeString( id_step, "result_name" ); resultCodeFieldName = rep.getStepAttributeString( id_step, "result_code" ); responseTimeFieldName = rep.getStepAttributeString( id_step, "response_time" ); + responseHeaderFieldName = rep.getStepAttributeString( id_step, "response_header" ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "HTTPPOSTMeta.Exception.UnexpectedErrorReadingStepInfo" ), e ); @@ -560,6 +573,7 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform rep.saveStepAttribute( id_transformation, id_step, "result_name", fieldName ); rep.saveStepAttribute( id_transformation, id_step, "result_code", resultCodeFieldName ); rep.saveStepAttribute( id_transformation, id_step, "response_time", responseTimeFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "response_header", responseHeaderFieldName ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "HTTPPOSTMeta.Exception.UnableToSaveStepInfo" ) + id_step, e ); @@ -732,4 +746,12 @@ public String getResponseTimeFieldName() { public void setResponseTimeFieldName( String responseTimeFieldName ) { this.responseTimeFieldName = responseTimeFieldName; } + + public String getResponseHeaderFieldName() { + return responseHeaderFieldName; + } + + public void setResponseHeaderFieldName( String responseHeaderFieldName ) { + this.responseHeaderFieldName = responseHeaderFieldName; + } } diff --git a/engine/src/org/pentaho/di/trans/steps/httppost/messages/messages_en_US.properties b/engine/src/org/pentaho/di/trans/steps/httppost/messages/messages_en_US.properties index 0221d1426e19..6ea700cca7c7 100644 --- a/engine/src/org/pentaho/di/trans/steps/httppost/messages/messages_en_US.properties +++ b/engine/src/org/pentaho/di/trans/steps/httppost/messages/messages_en_US.properties @@ -62,7 +62,8 @@ HTTPPOSTMeta.CheckResult.MissingArguments=Missing arguments, not found in input HTTPPOST.Log.NoField=URL field name is missing\! HTTPPOSTMeta.CheckResult.UrlfieldOk=URL field is specified. HTTPPOSTDialog.Result.Label=Result field name -HTTPPOSTDialog.ResponseTime.Label=Response time (milliseconds) field name +HTTPPOSTDialog.ResponseTime.Label=Response time (milliseconds) field name +HTTPPOSTDialog.ResponseHeader.Label=Response header field name HTTPPOSTDialog.ResultCode.Label=HTTP status code field name HTTPPOSTMeta.CheckResult.WrongTypeArguments=\ (found but wrong type\: {0} vs. {1}) HTTPPOSTDialog.AutoCommit.Label=Enable auto commit diff --git a/engine/src/org/pentaho/di/trans/steps/rest/Rest.java b/engine/src/org/pentaho/di/trans/steps/rest/Rest.java index 92686406ea51..c20d976394c5 100644 --- a/engine/src/org/pentaho/di/trans/steps/rest/Rest.java +++ b/engine/src/org/pentaho/di/trans/steps/rest/Rest.java @@ -30,15 +30,18 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; +import java.util.List; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManagerFactory; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.UriBuilder; import org.apache.commons.httpclient.auth.AuthScope; +import org.json.simple.JSONObject; import org.pentaho.di.core.Const; import org.pentaho.di.core.encryption.Encr; import org.pentaho.di.core.exception.KettleException; @@ -202,11 +205,25 @@ private Object[] callRest( Object[] rowData ) throws KettleException { // Get Response String body; + String headerString = null; try { body = response.getEntity( String.class ); } catch ( UniformInterfaceException ex ) { body = ""; } + // get Header + MultivaluedMap headers = searchForHeaders( response ); + JSONObject json = new JSONObject(); + for ( java.util.Map.Entry> entry : headers.entrySet() ) { + String name = entry.getKey(); + List value = entry.getValue(); + if ( value.size() > 1 ) { + json.put( name, value ); + } else { + json.put( name, value.get( 0 ) ); + } + } + headerString = json.toJSONString(); // for output int returnFieldsOffset = data.inputRowMeta.size(); // add response to output @@ -225,6 +242,10 @@ private Object[] callRest( Object[] rowData ) throws KettleException { if ( !Const.isEmpty( data.resultResponseFieldName ) ) { newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( responseTime ) ); } + // add response header to output + if ( !Const.isEmpty( data.resultHeaderFieldName ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString.toString() ); + } } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.CanNotReadURL", data.realUrl ), e ); @@ -314,6 +335,10 @@ public boolean verify( String hostname, SSLSession session ) { } } + + protected MultivaluedMap searchForHeaders( ClientResponse response ) { + return response.getHeaders(); + } public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { meta = (RestMeta) smi; data = (RestData) sdi; @@ -490,6 +515,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { data.resultFieldName = environmentSubstitute( meta.getFieldName() ); data.resultCodeFieldName = environmentSubstitute( meta.getResultCodeFieldName() ); data.resultResponseFieldName = environmentSubstitute( meta.getResponseTimeFieldName() ); + data.resultHeaderFieldName = environmentSubstitute( meta.getResponseHeaderFieldName() ); // get authentication settings once data.realProxyHost = environmentSubstitute( meta.getProxyHost() ); diff --git a/engine/src/org/pentaho/di/trans/steps/rest/RestData.java b/engine/src/org/pentaho/di/trans/steps/rest/RestData.java index 6beb14ea19cf..8abe2a35d6f7 100644 --- a/engine/src/org/pentaho/di/trans/steps/rest/RestData.java +++ b/engine/src/org/pentaho/di/trans/steps/rest/RestData.java @@ -73,6 +73,7 @@ public class RestData extends BaseStepData implements StepDataInterface { public String resultFieldName; public String resultCodeFieldName; public String resultResponseFieldName; + public String resultHeaderFieldName; /** Flag set headers **/ public boolean useHeaders; @@ -110,6 +111,7 @@ public RestData() { this.resultFieldName = null; this.resultCodeFieldName = null; this.resultResponseFieldName = null; + this.resultHeaderFieldName = null; this.nrheader = 0; this.nrParams = 0; this.nrMatrixParams = 0; diff --git a/engine/src/org/pentaho/di/trans/steps/rest/RestMeta.java b/engine/src/org/pentaho/di/trans/steps/rest/RestMeta.java index f9583e279607..21f2ca41847e 100644 --- a/engine/src/org/pentaho/di/trans/steps/rest/RestMeta.java +++ b/engine/src/org/pentaho/di/trans/steps/rest/RestMeta.java @@ -33,6 +33,7 @@ import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.exception.KettleXMLException; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.value.ValueMetaInteger; import org.pentaho.di.core.row.value.ValueMetaString; @@ -105,6 +106,7 @@ public class RestMeta extends BaseStepMeta implements StepMetaInterface { private String fieldName; private String resultCodeFieldName; private String responseTimeFieldName; + private String responseHeaderFieldName; /** proxy **/ private String proxyHost; @@ -411,6 +413,7 @@ public void setDefault() { this.fieldName = "result"; this.resultCodeFieldName = ""; this.responseTimeFieldName = ""; + this.responseHeaderFieldName = ""; this.method = HTTP_METHOD_GET; this.dynamicMethod = false; this.methodFieldName = null; @@ -441,6 +444,13 @@ public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterf v.setOrigin( name ); inputRowMeta.addValueMeta( v ); } + String headerFieldName = space.environmentSubstitute( responseHeaderFieldName ); + if ( !Const.isEmpty( headerFieldName ) ) { + ValueMetaInterface v = + new ValueMeta( headerFieldName, ValueMeta.TYPE_STRING ); + v.setOrigin( name ); + inputRowMeta.addValueMeta( v ); + } } public String getXML() { @@ -497,6 +507,7 @@ public String getXML() { retval.append( " " ).append( XMLHandler.addTagValue( "name", fieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "code", resultCodeFieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "response_time", responseTimeFieldName ) ); + retval.append( " " ).append( XMLHandler.addTagValue( "response_header", responseHeaderFieldName ) ); retval.append( " " ).append( Const.CR ); return retval.toString(); @@ -551,6 +562,7 @@ private void readData( Node stepnode, List data fieldName = XMLHandler.getTagValue( stepnode, "result", "name" ); // Optional, can be null resultCodeFieldName = XMLHandler.getTagValue( stepnode, "result", "code" ); // Optional, can be null responseTimeFieldName = XMLHandler.getTagValue( stepnode, "result", "response_time" ); // Optional, can be null + responseHeaderFieldName = XMLHandler.getTagValue( stepnode, "result", "response_header" ); // Optional, can be null } catch ( Exception e ) { throw new KettleXMLException( BaseMessages.getString( PKG, "RestMeta.Exception.UnableToReadStepInfo" ), e ); } @@ -600,6 +612,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis fieldName = rep.getStepAttributeString( id_step, "result_name" ); resultCodeFieldName = rep.getStepAttributeString( id_step, "result_code" ); responseTimeFieldName = rep.getStepAttributeString( id_step, "response_time" ); + responseHeaderFieldName = rep.getStepAttributeString( id_step, "response_header" ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "RestMeta.Exception.UnexpectedErrorReadingStepInfo" ), e ); @@ -644,6 +657,7 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform rep.saveStepAttribute( id_transformation, id_step, "result_name", fieldName ); rep.saveStepAttribute( id_transformation, id_step, "result_code", resultCodeFieldName ); rep.saveStepAttribute( id_transformation, id_step, "response_time", responseTimeFieldName ); + rep.saveStepAttribute( id_transformation, id_step, "response_header", responseHeaderFieldName ); } catch ( Exception e ) { throw new KettleException( BaseMessages.getString( PKG, "RestMeta.Exception.UnableToSaveStepInfo" ) + id_step, e ); @@ -877,6 +891,13 @@ public String getResponseTimeFieldName() { public void setResponseTimeFieldName( String responseTimeFieldName ) { this.responseTimeFieldName = responseTimeFieldName; } + public String getResponseHeaderFieldName() { + return responseHeaderFieldName; + } + + public void setResponseHeaderFieldName( String responseHeaderFieldName ) { + this.responseHeaderFieldName = responseHeaderFieldName; + } public static boolean isActiveBody( String method ) { if ( Const.isEmpty( method ) ) { diff --git a/engine/src/org/pentaho/di/trans/steps/rest/messages/messages_en_US.properties b/engine/src/org/pentaho/di/trans/steps/rest/messages/messages_en_US.properties index 5d333671834e..eb3398fce53e 100644 --- a/engine/src/org/pentaho/di/trans/steps/rest/messages/messages_en_US.properties +++ b/engine/src/org/pentaho/di/trans/steps/rest/messages/messages_en_US.properties @@ -70,7 +70,8 @@ Rest.Log.Encoding=Header content encoding \: {0} RestDialog.ColumnInfo.ParameterField=Parameter RestMeta.Exception.UnexpectedErrorReadingStepInfo=Unexpected error reading step information from the repository Rest.Error.KeyManagementException=Key management error -RestDialog.ResponseTime.Label=Response time (milliseconds) field name +RestDialog.ResponseTime.Label=Response time (milliseconds) field name +RestDialog.ResponseHeader.Label=Response header field name Rest.ErrorInStepRunning=Because of an error, this step can''t continue\: RestDialog.ProxyGroup.Label=Proxy to use RestDialog.SettingsGroup.Label=Settings diff --git a/engine/test-src/org/pentaho/di/trans/steps/http/HTTPIT.java b/engine/test-src/org/pentaho/di/trans/steps/http/HTTPIT.java index 3f42c3270600..e5aec0008d89 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/http/HTTPIT.java +++ b/engine/test-src/org/pentaho/di/trans/steps/http/HTTPIT.java @@ -23,29 +23,31 @@ package org.pentaho.di.trans.steps.http; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.pentaho.di.core.util.Assert.assertTrue; - import java.io.IOException; +import java.io.InputStreamReader; import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpMethod; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.InjectMocks; +import org.mockito.Mockito; import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.logging.LoggingObjectInterface; +import org.pentaho.di.core.row.RowMeta; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaInteger; +import org.pentaho.di.core.row.value.ValueMetaString; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.StepDataInterface; @@ -67,12 +69,15 @@ public class HTTPIT { private class HTTPHandler extends HTTP { - Object[] row = new Object[] { "anyData" }; + Object[] row; Object[] outputRow; + boolean override = false; public HTTPHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, - Trans trans ) { + Trans trans, boolean override ) { super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); + this.row = new Object[] { "anyData" }; + this.override = override; } /** @@ -88,11 +93,11 @@ public Object[] getRow() throws KettleException { * putRow is used to copy a row, to the alternate rowset(s) This should get priority over everything else! * (synchronized) If distribute is true, a row is copied only once to the output rowsets, otherwise copies are sent * to each rowset! - * + * * @param row * The row to put to the destination rowset(s). * @throws org.pentaho.di.core.exception.KettleStepException - * + * */ @Override public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { @@ -103,11 +108,47 @@ public Object[] getOutputRow() { return outputRow; } + + + @Override + protected int requestStatusCode( HttpMethod method, HostConfiguration hostConfiguration, HttpClient httpClient) + throws IOException { + if ( override ) { + return 402; + } else { + return super.requestStatusCode( method, hostConfiguration, httpClient); + } + } + + @Override + protected InputStreamReader openStream( String encoding, HttpMethod method ) throws Exception { + if ( override ) { + InputStreamReader mockInputStreamReader = Mockito.mock( InputStreamReader.class ); + when( mockInputStreamReader.read() ).thenReturn( -1 ); + return mockInputStreamReader; + } else { + return super.openStream( encoding, method ); + } + } + + @Override + protected Header[] searchForHeaders( HttpMethod method ) { + Header[] headers = { new Header( "host", host ) }; + if ( override ) { + return headers; + } else { + return super.searchForHeaders( method ); + } + } + + } + public static final String host = "localhost"; public static final int port = 9998; public static final String HTTP_LOCALHOST_9998 = "http://localhost:9998/"; + @InjectMocks private StepMockHelper stepMockHelper; private HttpServer httpServer; @@ -133,11 +174,43 @@ public void tearDown() throws Exception { } + @Test public void test204Answer() throws Exception { HTTPData data = new HTTPData(); - Object[] expectedRow = new Object[] { "", 204L, null, null, null, null, null, null, null, null, null, null }; - HTTP http = new HTTPHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans ); + int[] index = { 0, 1 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + Object[] expectedRow = new Object[] { "", 204L }; + HTTP http = + new HTTPHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false ); + RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); + http.setInputRowMeta( inputRowMeta ); + when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); + when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 ); + when( stepMockHelper.processRowsStepMetaInterface.getHeaderField() ).thenReturn( new String[] {} ); + when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} ); + when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" ); + when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" ); + http.init( stepMockHelper.processRowsStepMetaInterface, data ); + assertTrue( http.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); + Object[] out = ( (HTTPHandler) http ).getOutputRow(); + assertTrue( meta.equals( out, expectedRow, index ) ); + } + + @Test + public void testResponseHeader() throws Exception { + HTTPData data = new HTTPData(); + int[] index = { 0, 1, 2 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + meta.addValueMeta( new ValueMetaString( "headerFieldName" ) ); + Object[] expectedRow = + new Object[] { "", 402L, "{\"host\":\"localhost\"}" }; + HTTP http = + new HTTPHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, true ); RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); http.setInputRowMeta( inputRowMeta ); when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); @@ -146,12 +219,14 @@ public void test204Answer() throws Exception { when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} ); when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" ); when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" ); + when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF8" ); + when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn( + "ResponseHeaderFieldName" ); http.init( stepMockHelper.processRowsStepMetaInterface, data ); assertTrue( http.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); - System.out.println( Arrays.toString( expectedRow ) ); Object[] out = ( (HTTPHandler) http ).getOutputRow(); - System.out.println( Arrays.toString( out ) ); - assertTrue( Arrays.equals( expectedRow, out ) ); + assertTrue( meta.equals( out, expectedRow, index ) ); + } private void startHttp204Answer() throws IOException { @@ -172,8 +247,7 @@ public void testLoadSaveRoundTrip() throws KettleException { Arrays.asList( "url", "urlInField", "urlField", "encoding", "httpLogin", "httpPassword", "proxyHost", "proxyPort", "socketTimeout", "connectionTimeout", "closeIdleConnectionsTime", "argumentField", "argumentParameter", "headerField", "headerParameter", "fieldName", "resultCodeFieldName", - "responseTimeFieldName" ); - + "responseTimeFieldName", "responseHeaderFieldName" ); Map> fieldLoadSaveValidatorAttributeMap = new HashMap>(); diff --git a/engine/test-src/org/pentaho/di/trans/steps/httppost/HTTPPOSTIT.java b/engine/test-src/org/pentaho/di/trans/steps/httppost/HTTPPOSTIT.java index 6ee3900f8248..d1d421d61944 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/httppost/HTTPPOSTIT.java +++ b/engine/test-src/org/pentaho/di/trans/steps/httppost/HTTPPOSTIT.java @@ -30,22 +30,31 @@ import static org.pentaho.di.core.util.Assert.assertTrue; import java.io.IOException; +import java.io.InputStreamReader; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.methods.PostMethod; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.InjectMocks; +import org.mockito.Mockito; import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.logging.LoggingObjectInterface; +import org.pentaho.di.core.row.RowMeta; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaInteger; +import org.pentaho.di.core.row.value.ValueMetaString; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.StepDataInterface; @@ -70,12 +79,15 @@ private class HTTPPOSTHandler extends HTTPPOST { Object[] row = new Object[] { "anyData" }; Object[] outputRow; + boolean override; public HTTPPOSTHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, - Trans trans ) { + Trans trans, boolean override ) { super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); + this.override = override; } + /** * In case of getRow, we receive data from previous steps through the input rowset. In case we split the stream, we * have to copy the data to the alternate splits: rowsets 1 through n. @@ -104,6 +116,38 @@ public Object[] getOutputRow() { return outputRow; } + + @Override + protected int requestStatusCode( PostMethod post, HostConfiguration hostConfiguration, HttpClient httpPostClient) + throws IOException { + if ( override ) { + return 402; + } else { + return super.requestStatusCode( post, hostConfiguration, httpPostClient); + } + + } + + @Override + protected InputStreamReader openStream( String encoding, PostMethod post ) throws Exception { + if ( override ) { + InputStreamReader mockInputStreamReader = Mockito.mock( InputStreamReader.class ); + when( mockInputStreamReader.read() ).thenReturn( -1 ); + return mockInputStreamReader; + } else { + return super.openStream( encoding, post ); + } + } + + @Override + protected Header[] searchForHeaders( PostMethod post ) { + Header[] headers = { new Header( "host", host ) }; + if ( override ) { + return headers; + } else { + return super.searchForHeaders( post ); + } + } } public static final String host = "localhost"; @@ -139,9 +183,39 @@ public void tearDown() throws Exception { @Test public void test204Answer() throws Exception { HTTPPOSTData data = new HTTPPOSTData(); - Object[] expectedRow = new Object[] { "", 204L, null, null, null, null, null, null, null, null, null, null }; - HTTPPOST HTTPPOST = - new HTTPPOSTHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans ); + int[] index = { 0, 1 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + Object[] expectedRow = new Object[] { "", 204L }; + HTTPPOST HTTPPOST = new HTTPPOSTHandler( + stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false ); + RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); + HTTPPOST.setInputRowMeta( inputRowMeta ); + when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); + when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 ); + when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} ); + when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} ); + when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" ); + when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" ); + HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data ); + assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); + Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow(); + assertTrue( meta.equals( out, expectedRow, index ) ); + } + + @Test + public void testResponseHeader() throws Exception { + HTTPPOSTData data = new HTTPPOSTData(); + int[] index = { 0, 1, 2 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + meta.addValueMeta( new ValueMetaString( "headerFieldName" ) ); + Object[] expectedRow = + new Object[] { "", 402L, "{\"host\":\"localhost\"}" }; + HTTPPOST HTTPPOST = new HTTPPOSTHandler( + stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, true ); RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); HTTPPOST.setInputRowMeta( inputRowMeta ); when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); @@ -150,14 +224,16 @@ public void test204Answer() throws Exception { when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} ); when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" ); when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" ); + when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF8" ); + when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn( + "ResponseHeaderFieldName" ); HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data ); assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); - System.out.println( Arrays.toString( expectedRow ) ); Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow(); - System.out.println( Arrays.toString( out ) ); - assertTrue( Arrays.equals( expectedRow, out ) ); + assertTrue( meta.equals( out, expectedRow, index ) ); } + private void startHttp204Answer() throws IOException { httpServer = HttpServer.create( new InetSocketAddress( HTTPPOSTIT.host, HTTPPOSTIT.port ), 10 ); httpServer.createContext( "/", new HttpHandler() { @@ -176,7 +252,7 @@ public void testLoadSaveRoundTrip() throws KettleException { Arrays.asList( "postAFile", "encoding", "url", "urlInField", "urlField", "requestEntity", "httpLogin", "httpPassword", "proxyHost", "proxyPort", "socketTimeout", "connectionTimeout", "closeIdleConnectionsTime", "argumentField", "argumentParameter", "argumentHeader", "queryField", - "queryParameter", "fieldName", "resultCodeFieldName", "responseTimeFieldName" ); + "queryParameter", "fieldName", "resultCodeFieldName", "responseTimeFieldName", "responseHeaderFieldName" ); Map> fieldLoadSaveValidatorAttributeMap = new HashMap>(); diff --git a/engine/test-src/org/pentaho/di/trans/steps/rest/RestIT.java b/engine/test-src/org/pentaho/di/trans/steps/rest/RestIT.java index f2b71f3d81f9..43cbc40e3af7 100644 --- a/engine/test-src/org/pentaho/di/trans/steps/rest/RestIT.java +++ b/engine/test-src/org/pentaho/di/trans/steps/rest/RestIT.java @@ -29,20 +29,22 @@ import static org.mockito.Mockito.when; import static org.pentaho.di.core.util.Assert.assertTrue; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import com.sun.jersey.api.client.ClientResponse; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Mockito; import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleStepException; import org.pentaho.di.core.logging.LoggingObjectInterface; +import org.pentaho.di.core.row.RowMeta; import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaInteger; +import org.pentaho.di.core.row.value.ValueMetaString; import org.pentaho.di.trans.Trans; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.StepDataInterface; @@ -56,6 +58,8 @@ import com.sun.jersey.api.container.httpserver.HttpServerFactory; import com.sun.net.httpserver.HttpServer; +import javax.ws.rs.core.MultivaluedMap; + /** * User: Dzmitry Stsiapanau Date: 11/29/13 Time: 3:42 PM */ @@ -67,10 +71,12 @@ private class RestHandler extends Rest { Object[] row = new Object[] { "anyData" }; Object[] outputRow; + boolean override; public RestHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, - Trans trans ) { + Trans trans, boolean override ) { super( stepMeta, stepDataInterface, copyNr, transMeta, trans ); + this.override = override; } @SuppressWarnings( "unused" ) @@ -102,6 +108,25 @@ public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepEx outputRow = row; } + @Override + protected MultivaluedMap searchForHeaders( ClientResponse response ) { + if ( override ) { + String host = "host"; + List localhost = new ArrayList(); + localhost.add( "localhost" ); + Map.Entry> entry = Mockito.mock( Map.Entry.class ); + when( entry.getKey() ).thenReturn( host ); + when( entry.getValue() ).thenReturn( localhost ); + Set>> set = new HashSet>>(); + set.add( entry ); + MultivaluedMap test = Mockito.mock( MultivaluedMap.class ); + when( test.entrySet() ).thenReturn( set ); + return test; + } else { + return super.searchForHeaders( response ); + } + } + public Object[] getOutputRow() { return outputRow; } @@ -135,8 +160,13 @@ public void tearDown() throws Exception { @Test public void testNoContent() throws Exception { RestData data = new RestData(); - Object[] expectedRow = new Object[] { "", 204L, null, null, null, null, null, null, null, null, null, null }; - Rest rest = new RestHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans ); + int[] index = { 0, 1 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + Object[] expectedRow = new Object[] { "", 204L }; + Rest rest = + new RestHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false ); RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); rest.setInputRowMeta( inputRowMeta ); when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); @@ -147,19 +177,47 @@ public void testNoContent() throws Exception { data.resultFieldName = "ResultFieldName"; data.resultCodeFieldName = "ResultCodeFieldName"; assertTrue( rest.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); - System.out.println( Arrays.toString( expectedRow ) ); Object[] out = ( (RestHandler) rest ).getOutputRow(); - System.out.println( Arrays.toString( out ) ); - assertTrue( Arrays.equals( expectedRow, out ) ); + assertTrue( meta.equals( out, expectedRow, index ) ); } + @Test + public void testResponseHeader() throws Exception { + RestData data = new RestData(); + int[] index = { 0, 1, 2 }; + RowMeta meta = new RowMeta(); + meta.addValueMeta( new ValueMetaString( "fieldName" ) ); + meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) ); + meta.addValueMeta( new ValueMetaString( "headerFieldName" ) ); + Object[] expectedRow = + new Object[] { "", 204L, "{\"host\":\"localhost\"}" }; + Rest rest = + new RestHandler( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, true ); + RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); + rest.setInputRowMeta( inputRowMeta ); + when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); + when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( + HTTP_LOCALHOST_9998 + "restTest/restNoContentAnswer" ); + when( stepMockHelper.processRowsStepMetaInterface.getMethod() ).thenReturn( RestMeta.HTTP_METHOD_GET ); + when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn( + "ResponseHeaderFieldName" ); + rest.init( stepMockHelper.processRowsStepMetaInterface, data ); + data.resultFieldName = "ResultFieldName"; + data.resultCodeFieldName = "ResultCodeFieldName"; + assertTrue( rest.processRow( stepMockHelper.processRowsStepMetaInterface, data ) ); + Object[] out = ( (RestHandler) rest ).getOutputRow(); + assertTrue( meta.equals( out, expectedRow, index ) ); + } + + @Test public void testLoadSaveRoundTrip() throws KettleException { List attributes = Arrays.asList( "applicationType", "method", "url", "urlInField", "dynamicMethod", "methodFieldName", "urlField", "bodyField", "httpLogin", "httpPassword", "proxyHost", "proxyPort", "preemptive", "trustStoreFile", "trustStorePassword", "headerField", "headerName", "parameterField", "parameterName", - "matrixParameterField", "matrixParameterName", "fieldName", "resultCodeFieldName", "responseTimeFieldName" ); + "matrixParameterField", "matrixParameterName", "fieldName", "resultCodeFieldName", "responseTimeFieldName", + "responseHeaderFieldName" ); Map> fieldLoadSaveValidatorAttributeMap = new HashMap>(); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/http/HTTPDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/http/HTTPDialog.java index 7cf7b2d0ce95..b5d09a9e654c 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/http/HTTPDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/http/HTTPDialog.java @@ -133,6 +133,10 @@ public class HTTPDialog extends BaseStepDialog implements StepDialogInterface { private TextVar wResponseTime; private FormData fdlResponseTime, fdResponseTime; + private Label wlResponseHeader; + private TextVar wResponseHeader; + private FormData fdlResponseHeader, fdResponseHeader; + private HTTPMeta input; private ColumnInfo[] colinf; @@ -475,6 +479,23 @@ public void focusGained( org.eclipse.swt.events.FocusEvent e ) { fdResponseTime.top = new FormAttachment( wResultCode, margin ); fdResponseTime.right = new FormAttachment( 100, 0 ); wResponseTime.setLayoutData( fdResponseTime ); + // Response header line... + wlResponseHeader = new Label( gOutputFields, SWT.RIGHT ); + wlResponseHeader.setText( BaseMessages.getString( PKG, "HTTPDialog.ResponseHeader.Label" ) ); + props.setLook( wlResponseHeader ); + fdlResponseHeader = new FormData(); + fdlResponseHeader.left = new FormAttachment( 0, 0 ); + fdlResponseHeader.right = new FormAttachment( middle, -margin ); + fdlResponseHeader.top = new FormAttachment( wResponseTime, margin ); + wlResponseHeader.setLayoutData( fdlResponseHeader ); + wResponseHeader = new TextVar( transMeta, gOutputFields, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wResponseHeader ); + wResponseHeader.addModifyListener( lsMod ); + fdResponseHeader = new FormData(); + fdResponseHeader.left = new FormAttachment( middle, 0 ); + fdResponseHeader.top = new FormAttachment( wResponseTime, margin ); + fdResponseHeader.right = new FormAttachment( 100, 0 ); + wResponseHeader.setLayoutData( fdResponseHeader ); FormData fdOutputFields = new FormData(); fdOutputFields.left = new FormAttachment( 0, 0 ); @@ -929,6 +950,9 @@ public void getData() { if ( input.getResponseTimeFieldName() != null ) { wResponseTime.setText( input.getResponseTimeFieldName() ); } + if ( input.getResponseHeaderFieldName() != null ) { + wResponseHeader.setText( input.getResponseHeaderFieldName() ); + } wFields.setRowNums(); wFields.optWidth( true ); @@ -986,6 +1010,7 @@ private void ok() { input.setProxyPort( wProxyPort.getText() ); input.setResultCodeFieldName( wResultCode.getText() ); input.setResponseTimeFieldName( wResponseTime.getText() ); + input.setResponseHeaderFieldName( wResponseHeader.getText() ); input.setSocketTimeout( wSocketTimeOut.getText() ); input.setConnectionTimeout( wConnectionTimeOut.getText() ); input.setCloseIdleConnectionsTime( wCloseIdleConnectionsTime.getText() ); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/httppost/HTTPPOSTDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/httppost/HTTPPOSTDialog.java index fda0e501c230..33679fbc7e08 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/httppost/HTTPPOSTDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/httppost/HTTPPOSTDialog.java @@ -97,6 +97,9 @@ public class HTTPPOSTDialog extends BaseStepDialog implements StepDialogInterfac private Label wlResponseTime; private TextVar wResponseTime; private FormData fdlResponseTime, fdResponseTime; + private Label wlResponseHeader; + private TextVar wResponseHeader; + private FormData fdlResponseHeader, fdResponseHeader; private Label wlFields; private TableView wFields; @@ -532,6 +535,23 @@ public void focusGained( org.eclipse.swt.events.FocusEvent e ) { fdResponseTime.top = new FormAttachment( wResultCode, margin ); fdResponseTime.right = new FormAttachment( 100, 0 ); wResponseTime.setLayoutData( fdResponseTime ); + // Response header line... + wlResponseHeader = new Label( gOutputFields, SWT.RIGHT ); + wlResponseHeader.setText( BaseMessages.getString( PKG, "HTTPPOSTDialog.ResponseHeader.Label" ) ); + props.setLook( wlResponseHeader ); + fdlResponseHeader = new FormData(); + fdlResponseHeader.left = new FormAttachment( 0, 0 ); + fdlResponseHeader.right = new FormAttachment( middle, -margin ); + fdlResponseHeader.top = new FormAttachment( wResponseTime, margin ); + wlResponseHeader.setLayoutData( fdlResponseHeader ); + wResponseHeader = new TextVar( transMeta, gOutputFields, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wResponseHeader ); + wResponseHeader.addModifyListener( lsMod ); + fdResponseHeader = new FormData(); + fdResponseHeader.left = new FormAttachment( middle, 0 ); + fdResponseHeader.top = new FormAttachment( wResponseTime, margin ); + fdResponseHeader.right = new FormAttachment( 100, 0 ); + wResponseHeader.setLayoutData( fdResponseHeader ); FormData fdOutputFields = new FormData(); fdOutputFields.left = new FormAttachment( 0, 0 ); @@ -1019,6 +1039,9 @@ public void getData() { if ( input.getProxyPort() != null ) { wProxyPort.setText( input.getProxyPort() ); } + if ( input.getResponseHeaderFieldName() != null ) { + wResponseHeader.setText( input.getResponseHeaderFieldName() ); + } wSocketTimeOut.setText( Const.NVL( input.getSocketTimeout(), "" ) ); wConnectionTimeOut.setText( Const.NVL( input.getConnectionTimeout(), "" ) ); @@ -1077,6 +1100,7 @@ private void ok() { input.setFieldName( wResult.getText() ); input.setResultCodeFieldName( wResultCode.getText() ); input.setResponseTimeFieldName( wResponseTime.getText() ); + input.setResponseHeaderFieldName( wResponseHeader.getText() ); input.setEncoding( wEncoding.getText() ); input.setPostAFile( wPostAFile.getSelection() ); input.setHttpLogin( wHttpLogin.getText() ); diff --git a/ui/src/org/pentaho/di/ui/trans/steps/rest/RestDialog.java b/ui/src/org/pentaho/di/ui/trans/steps/rest/RestDialog.java index 82c411d59650..7991ffc8ddee 100644 --- a/ui/src/org/pentaho/di/ui/trans/steps/rest/RestDialog.java +++ b/ui/src/org/pentaho/di/ui/trans/steps/rest/RestDialog.java @@ -169,6 +169,9 @@ public class RestDialog extends BaseStepDialog implements StepDialogInterface { private Label wlResponseTime; private TextVar wResponseTime; private FormData fdlResponseTime, fdResponseTime; + private Label wlResponseHeader; + private TextVar wResponseHeader; + private FormData fdlResponseHeader, fdResponseHeader; private Label wlTrustStorePassword; private TextVar wTrustStorePassword; @@ -551,6 +554,23 @@ public void widgetSelected( SelectionEvent e ) { fdResponseTime.top = new FormAttachment( wResultCode, margin ); fdResponseTime.right = new FormAttachment( 100, 0 ); wResponseTime.setLayoutData( fdResponseTime ); + // Response header line... + wlResponseHeader = new Label( gOutputFields, SWT.RIGHT ); + wlResponseHeader.setText( BaseMessages.getString( PKG, "RestDialog.ResponseHeader.Label" ) ); + props.setLook( wlResponseHeader ); + fdlResponseHeader = new FormData(); + fdlResponseHeader.left = new FormAttachment( 0, 0 ); + fdlResponseHeader.right = new FormAttachment( middle, -margin ); + fdlResponseHeader.top = new FormAttachment( wResponseTime, margin ); + wlResponseHeader.setLayoutData( fdlResponseHeader ); + wResponseHeader = new TextVar( transMeta, gOutputFields, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); + props.setLook( wResponseHeader ); + wResponseHeader.addModifyListener( lsMod ); + fdResponseHeader = new FormData(); + fdResponseHeader.left = new FormAttachment( middle, 0 ); + fdResponseHeader.top = new FormAttachment( wResponseTime, margin ); + fdResponseHeader.right = new FormAttachment( 100, 0 ); + wResponseHeader.setLayoutData( fdResponseHeader ); FormData fdOutputFields = new FormData(); fdOutputFields.left = new FormAttachment( 0, 0 ); @@ -1306,6 +1326,9 @@ public void getData() { if ( input.getTrustStorePassword() != null ) { wTrustStorePassword.setText( input.getTrustStorePassword() ); } + if ( input.getResponseHeaderFieldName() != null ) { + wResponseHeader.setText( input.getResponseHeaderFieldName() ); + } wApplicationType.setText( Const.NVL( input.getApplicationType(), "" ) ); @@ -1365,6 +1388,7 @@ private void ok() { input.setFieldName( wResult.getText() ); input.setResultCodeFieldName( wResultCode.getText() ); input.setResponseTimeFieldName( wResponseTime.getText() ); + input.setResponseHeaderFieldName( wResponseHeader.getText() ); input.setHttpLogin( wHttpLogin.getText() ); input.setHttpPassword( wHttpPassword.getText() );