- added warnings for failed transferRWI (dht-in)

- fixed parseMultipart (uncompress gzipped body) (dht-in)
- fixed parseMultipart (using content-length only if uncompressed)
- better gzipped POST (chunked instead of content-length) (dht-out)



git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5096 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/transferRWI.java

@@ -38,6 +38,7 @@
 import de.anomic.server.serverCore;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
+import de.anomic.server.logging.serverLog;
 import de.anomic.tools.nxTools;
 import de.anomic.xml.RSSFeed;
 import de.anomic.xml.RSSMessage;
@@ -53,10 +54,23 @@ public static serverObjects respond(final httpRequestHeader header, final server
         // return variable that accumulates replacements
         final plasmaSwitchboard sb = (plasmaSwitchboard) env;
         final serverObjects prop = new serverObjects();
-        if ((post == null) || (env == null)) return prop;
-        if (!yacyNetwork.authentifyRequest(post, env)) return prop;
-        if (!post.containsKey("wordc")) return prop;
-        if (!post.containsKey("entryc")) return prop;
+        final String contentType = header.getContentType();
+        if ((post == null) || (env == null)) {
+            logWarning(contentType, "post or env is null!");
+            return prop;
+        }
+        if (!yacyNetwork.authentifyRequest(post, env)) {
+            logWarning(contentType, "not authentified");
+            return prop;
+        }
+        if (!post.containsKey("wordc")) {
+            logWarning(contentType, "missing wordc");
+            return prop;
+        }
+        if (!post.containsKey("entryc")) {
+            logWarning(contentType, "missing entryc");
+            return prop;
+        }
 
         // request values
         final String iam      = post.get("iam", "");                      // seed hash of requester
@@ -197,4 +211,12 @@ public static serverObjects respond(final httpRequestHeader header, final server
         // return rewrite properties
         return prop;
     }
+
+    /**
+     * @param requestIdentifier
+     * @param msg
+     */
+    private static void logWarning(final String requestIdentifier, final String msg) {
+        serverLog.logWarning("transferRWI", requestIdentifier +" "+ msg);
+    }
 }

source/de/anomic/http/JakartaCommonsHttpClient.java

@@ -323,6 +323,7 @@ public JakartaCommonsHttpResponse POST(final String uri, final List<Part> multip
             data = zipRequest(data);
 
             post.setRequestHeader(httpResponseHeader.CONTENT_ENCODING, httpHeader.CONTENT_ENCODING_GZIP);
+            post.setContentChunked(true);
         }
         post.setRequestEntity(data);
         // redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" -

source/de/anomic/http/httpd.java

@@ -47,6 +47,7 @@
 import java.util.Properties;
 import java.util.StringTokenizer;
 import java.util.Map.Entry;
+import java.util.zip.GZIPInputStream;
 
 import org.apache.commons.fileupload.FileItem;
 import org.apache.commons.fileupload.FileItemFactory;
@@ -877,16 +878,14 @@ public static String decodeHtmlEntities(String s) {
      * @param header
      *            hier muss ARGC gesetzt werden!
      * @param args
-     * @param in
+     * @param in the raw body
      * @return
      * @throws IOException
      */
     @SuppressWarnings("unchecked")
     public static HashMap<String, byte[]> parseMultipart(final httpRequestHeader header, final serverObjects args, final InputStream in)
             throws IOException {
-
-        final long clength = header.getContentLength();
-        final InputStream body = (clength > 0) ? new ContentLengthInputStream(in, clength) : in;
+        final InputStream body = prepareBody(header, in);
 
         RequestContext request = new yacyContextRequest(header, body);
 
@@ -908,7 +907,6 @@ public static HashMap<String, byte[]> parseMultipart(final httpRequestHeader hea
 
         // format information for further usage
         final HashMap<String, byte[]> files = new HashMap<String, byte[]>();
-        int formFieldCount = 0;
         for (FileItem item : items) {
             if (item.isFormField()) {
                 // simple text
@@ -919,7 +917,6 @@ public static HashMap<String, byte[]> parseMultipart(final httpRequestHeader hea
                     // use default encoding (given as header or ISO-8859-1)
                     args.put(item.getFieldName(), item.getString());
                 }
-                formFieldCount++;
             } else {
                 // file
                 args.put(item.getFieldName(), item.getName());
@@ -932,41 +929,72 @@ public static HashMap<String, byte[]> parseMultipart(final httpRequestHeader hea
 
         return files;
     }
+
+    /**
+     * prepares the body so that it can be read as whole plain text
+     * (uncompress if necessary and ensure correct ending)
+     * 
+     * @param header
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    private static InputStream prepareBody(final httpRequestHeader header, final InputStream in) throws IOException {
+        InputStream body = in;
+        // data may be compressed
+        final String bodyEncoding = header.get(httpHeader.CONTENT_ENCODING);
+        if(httpHeader.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding) && !(body instanceof GZIPInputStream)) {
+            System.out.println("+-+ DEBUG "+ header.getContentType() +" is GZIP");
+            body = new GZIPInputStream(body);
+            // length of uncompressed data is unknown
+            header.remove(httpHeader.CONTENT_LENGTH);
+        } else {
+            // ensure the end of data (if client keeps alive the connection)
+            final long clength = header.getContentLength();
+            if (clength > 0) {
+                System.out.println("+-+ DEBUG "+ header.getContentType() +" has length "+ clength);
+                body = new ContentLengthInputStream(body, clength);
+            } else {
+                System.out.println("+-+ DEBUG "+ header.getContentType() +" no length "+ clength);
+            }
+        }
+        return body;
+    }
 
     /**
      * wraps the request into a org.apache.commons.fileupload.RequestContext
      * 
-	 * @author danielr
-	 * @since 07.08.2008
-	 */
-	private static class yacyContextRequest extends httpRequestHeader implements RequestContext {
-		
-	    private static final long serialVersionUID = -8936741958551376593L;
-        
+     * @author danielr
+     * @since 07.08.2008
+     */
+    private static class yacyContextRequest extends httpRequestHeader implements RequestContext {
+
+        private static final long serialVersionUID = -8936741958551376593L;
+
         private final InputStream inStream;
-	
-		/**
-		 * creates a new yacyContextRequest
-		 * 
-		 * @param header
-		 * @param in
-		 */
-		public yacyContextRequest(Map<String, String> requestHeader, InputStream in) {
-		    super(null, requestHeader);
-			this.inStream = in;
-		}
-	
-		/*
-		 * (non-Javadoc)
-		 * 
-		 * @see org.apache.commons.fileupload.RequestContext#getInputStream()
-		 */
-		//@Override
-		public InputStream getInputStream() throws IOException {
-			return inStream;
-		}
-	
-	}
+
+        /**
+         * creates a new yacyContextRequest
+         * 
+         * @param header
+         * @param in
+         */
+        public yacyContextRequest(Map<String, String> requestHeader, InputStream in) {
+            super(null, requestHeader);
+            this.inStream = in;
+        }
+
+        /*
+         * (non-Javadoc)
+         * 
+         * @see org.apache.commons.fileupload.RequestContext#getInputStream()
+         */
+        // @Override
+        public InputStream getInputStream() throws IOException {
+            return inStream;
+        }
+
+    }
 
 	/*
 	static int nextPos = -1;