From f7891af8cb8606af26b9cba6af3d29bdacf8e3bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=B0=9A=E5=B7=9D?= <yang-shangchuan@qq.com>
Date: Wed, 17 Apr 2013 21:40:57 +0800
Subject: [PATCH 1/4] fix coding bug

---
 src/web/jsp/cached.jsp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/web/jsp/cached.jsp b/src/web/jsp/cached.jsp
index 6968bea0be..ccee8acef6 100644
--- a/src/web/jsp/cached.jsp
+++ b/src/web/jsp/cached.jsp
@@ -49,6 +49,10 @@
     // but I don't know how to emit 'byte sequence' in JSP.
     // out.getOutputStream().write(bean.getContent(details)) may work, 
     // but I'm not sure.
+
+    //fix bug：CharEncodingForConversion in ParseData's ParseMeta, not in ParseData's ContentMeta
+    metaData = bean.getParseData(details).getParseMeta();
+
     String encoding = (String) metaData.get("CharEncodingForConversion"); 
     if (encoding != null) {
       try {
@@ -59,8 +63,11 @@
         content = new String(bean.getContent(details), "windows-1252");
       }
     }
-    else 
-      content = new String(bean.getContent(details));
+    else {
+      ////fix bug：if http response Header Content-Type return wrong coding，then get coding from the original content of the page
+      encoding=org.apache.nutch.protocol.Content.getEncoding(bean.getContent(details), nutchConf.get("parser.character.encoding.default"));
+      content = new String(bean.getContent(details),encoding);
+    }
   }
 %>
 <!--

From d56898ffad3b52c2301cd06873e4366879187ba7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=B0=9A=E5=B7=9D?= <yang-shangchuan@qq.com>
Date: Wed, 17 Apr 2013 21:49:02 +0800
Subject: [PATCH 2/4] fix coding bug

---
 src/java/org/apache/nutch/protocol/Content.java | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/java/org/apache/nutch/protocol/Content.java b/src/java/org/apache/nutch/protocol/Content.java
index 79f6d87c39..95b8e2b015 100755
--- a/src/java/org/apache/nutch/protocol/Content.java
+++ b/src/java/org/apache/nutch/protocol/Content.java
@@ -235,6 +235,7 @@ public boolean equals(Object o) {
   }
 
   public String toString() {
+    String encoding=getEncoding();
     StringBuffer buffer = new StringBuffer();
 
     buffer.append("Version: " + version + "\n");
@@ -243,7 +244,16 @@ public String toString() {
     buffer.append("contentType: " + contentType + "\n");
     buffer.append("metadata: " + metadata + "\n");
     buffer.append("Content:\n");
-    buffer.append(new String(content)); // try default encoding
+    if(encoding!=null && !"".equals(encoding.trim())){
+      try {
+			buffer.append(new String(content,encoding));
+		} catch (UnsupportedEncodingException e) {
+			buffer.append(new String(content));
+			e.printStackTrace();
+		} 
+    }else{
+		buffer.append(new String(content));
+    }
 
     return buffer.toString();
 

From 82351b9345e354fe72c6a52d4b6455dd82f464b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=B0=9A=E5=B7=9D?= <yang-shangchuan@qq.com>
Date: Wed, 17 Apr 2013 21:52:23 +0800
Subject: [PATCH 3/4] fix coding bug

---
 .../org/apache/nutch/protocol/Content.java    | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/src/java/org/apache/nutch/protocol/Content.java b/src/java/org/apache/nutch/protocol/Content.java
index 95b8e2b015..c780cebf12 100755
--- a/src/java/org/apache/nutch/protocol/Content.java
+++ b/src/java/org/apache/nutch/protocol/Content.java
@@ -302,4 +302,61 @@ private String getContentType(String typeName, String url, byte[] data) {
     return this.mimeTypes.autoResolveContentType(typeName, url, data);
   }
 
+  /////
+  ////The following code is added to detect coding 
+  /////
+
+  
+  public String getEncoding(){
+	  return get(content,this,conf.get("parser.character.encoding.default"));
+  }
+  public static String getEncoding(byte[] data,String defaultEncoding){
+	  return get(data,new Content(),defaultEncoding);
+  }
+
+  private static String get(byte[] byteContent,Content content,String defaultEncoding){
+      EncodingDetector detector = new EncodingDetector(NutchConfiguration.create());
+      detector.addClue(sniffCharacterEncoding(byteContent), "sniffed");
+      detector.autoDetectClues(content, true);
+      String encoding = detector.guessEncoding(content, defaultEncoding);
+      return encoding;
+  }
+  
+  private static final int CHUNK_SIZE = 2000;
+  
+  private static Pattern metaPattern =
+	    Pattern.compile("<meta\\s+([^>]*http-equiv=\"?content-type\"?[^>]*)>",
+	                    Pattern.CASE_INSENSITIVE);
+  
+  private static Pattern charsetPattern =
+	    Pattern.compile("charset=\\s*([a-z][_\\-0-9a-z]*)",
+	                    Pattern.CASE_INSENSITIVE);
+
+  private static String sniffCharacterEncoding(byte[] content) {
+    int length = content.length < CHUNK_SIZE ? 
+                 content.length : CHUNK_SIZE;
+
+    // We don't care about non-ASCII parts so that it's sufficient
+    // to just inflate each byte to a 16-bit value by padding. 
+    // For instance, the sequence {0x41, 0x82, 0xb7} will be turned into 
+    // {U+0041, U+0082, U+00B7}. 
+    String str = "";
+    try {
+      str = new String(content, 0, length,
+                       Charset.forName("ASCII").toString());
+    } catch (UnsupportedEncodingException e) {
+      // code should never come here, but just in case... 
+      return null;
+    }
+
+    Matcher metaMatcher = metaPattern.matcher(str);
+    String encoding = null;
+    if (metaMatcher.find()) {
+      Matcher charsetMatcher = charsetPattern.matcher(metaMatcher.group(1));
+      if (charsetMatcher.find()) 
+        encoding = new String(charsetMatcher.group(1));
+    }
+
+    return encoding;
+  }
 }

From bf8518133c98e4a8220b684bb0f5fc6c3167589f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=B0=9A=E5=B7=9D?= <yang-shangchuan@qq.com>
Date: Wed, 17 Apr 2013 21:56:46 +0800
Subject: [PATCH 4/4] fix coding bug

---
 src/java/org/apache/nutch/util/EncodingDetector.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/java/org/apache/nutch/util/EncodingDetector.java b/src/java/org/apache/nutch/util/EncodingDetector.java
index b0440805cf..2b83f2e834 100644
--- a/src/java/org/apache/nutch/util/EncodingDetector.java
+++ b/src/java/org/apache/nutch/util/EncodingDetector.java
@@ -307,7 +307,14 @@ private void findDisagreements(String url, List<EncodingClue> newClues) {
   }
 
   public static String resolveEncodingAlias(String encoding) {
-    if (encoding == null || !Charset.isSupported(encoding))
+    //bug fix: wrong encoding ignored without throwing an exception
+  	boolean support=false;
+  	try{
+  		support=Charset.isSupported(encoding);
+  	}catch(Exception e){
+  		return null;
+  	}
+    if (encoding == null || !support)
       return null;
     String canonicalName = new String(Charset.forName(encoding).name());
     return ALIASES.containsKey(canonicalName) ? ALIASES.get(canonicalName)