Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'master' of github.com:zanata/zanata

  • Loading branch information...
commit 681bb9832327eda581fc1ddfa6e47afb8b64a0e6 2 parents 89a0609 + eaae695
Alex Eng aeng authored
65 common/zanata-adapter-po/src/main/java/org/zanata/adapter/po/PoReader2.java
View
@@ -3,13 +3,11 @@
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
-import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang.StringUtils;
import org.fedorahosted.tennera.jgettext.HeaderFields;
import org.fedorahosted.tennera.jgettext.Message;
@@ -27,6 +25,7 @@
import org.zanata.rest.dto.resource.TextFlow;
import org.zanata.rest.dto.resource.TextFlowTarget;
import org.zanata.rest.dto.resource.TranslationsResource;
+import org.zanata.util.HashUtil;
import org.zanata.util.ShortString;
import com.google.common.collect.ImmutableSet;
@@ -44,6 +43,16 @@ public PoReader2()
{
}
+ /**
+ * Extract contents of a PO file and convert to a TranslationsResource.
+ * NB: If the file contains the gettext header Content-Type, it must be
+ * set to ASCII, CHARSET, UTF8 or UTF-8, or an exception will occur.
+ * @param inputSource PO file to be extracted
+ * @param srcDoc source language document
+ * @param useSourceOrder ensure that the TextFlowTargets have the
+ * same order as the TextFlows in srcDoc
+ * @return converted PO file as TranslationsResource
+ */
public TranslationsResource extractTarget(InputSource inputSource, Resource srcDoc, boolean useSourceOrder)
{
TranslationsResource document = new TranslationsResource();
@@ -126,11 +135,37 @@ else if (message.isPlural())
return document;
}
+ /**
+ * Checks that the file is safe to read as UTF-8.
+ * @param hf
+ */
+ private static void checkContentType(HeaderFields hf)
+ {
+// @formatter:off
+ String contentType = hf.getValue(HeaderFields.KEY_ContentType);
+ if (contentType == null)
+ return;
+ String ct = contentType.toLowerCase();
+ if (!ct.contains("charset="))
+ return;
+ if (ct.contains("charset=charset") || ct.contains("charset=ascii") ||
+ ct.contains("charset=utf-8") || ct.contains("charset=utf8"))
+ {
+ return;
+ }
+ else
+ {
+ throw new RuntimeException("unsupported charset in " + HeaderFields.KEY_ContentType + ": " + contentType);
+ }
+// @formatter:on
+ }
+
private static void extractPotHeader(Message message, PoHeader potHeader)
{
potHeader.setComment(StringUtils.join(message.getComments(), "\n"));
HeaderFields hf = HeaderFields.wrap(message);
+ checkContentType(hf);
for (String key : hf.getKeys())
{
String val = hf.getValue(key);
@@ -148,6 +183,7 @@ private static void extractPoHeader(Message message, PoTargetHeader poHeader)
poHeader.setComment(StringUtils.join(message.getComments(), "\n"));
HeaderFields hf = HeaderFields.wrap(message);
+ checkContentType(hf);
for (String key : hf.getKeys())
{
String val = hf.getValue(key);
@@ -164,6 +200,15 @@ else if (!POT_HEADER_FIELDS.contains(key))
}
}
+ /**
+ * Extract contents of a POT file and convert to a Resource.
+ * NB: If the file contains the gettext header Content-Type, it must be
+ * set to ASCII, CHARSET, UTF8 or UTF-8, or an exception will occur.
+ * @param inputSource POT file to be extracted
+ * @param sourceLocaleId locale of POT, used to set metadata fields
+ * @param docName name of POT file (minus .pot extension) used to set metadata fields
+ * @return converted POT file as Resource
+ */
public Resource extractTemplate(InputSource inputSource, LocaleId sourceLocaleId, String docName)
{
Resource document = new Resource(docName);
@@ -283,21 +328,7 @@ static String createId(Message message)
{
String sep = "\u0000";
String hashBase = message.getMsgctxt() == null ? message.getMsgid() : message.getMsgctxt() + sep + message.getMsgid();
- return generateHash(hashBase);
- }
-
- public static String generateHash(String key)
- {
- try
- {
- MessageDigest md5 = MessageDigest.getInstance("MD5");
- md5.reset();
- return new String(Hex.encodeHex(md5.digest(key.getBytes("UTF-8"))));
- }
- catch (Exception exc)
- {
- throw new RuntimeException(exc);
- }
+ return HashUtil.generateHash(hashBase);
}
}
28 common/zanata-adapter-po/src/main/java/org/zanata/adapter/po/PoWriter2.java
View
@@ -73,7 +73,7 @@ public void writePotToDir(File potDir, Resource doc) throws IOException
File potFile = new File(potDir, doc.getName() + ".pot");
PathUtil.makeParents(potFile);
FileWriter fWriter = new FileWriter(potFile);
- write(fWriter, doc, null);
+ write(fWriter, "UTF-8", doc, null);
}
/**
@@ -93,7 +93,7 @@ public void writePo(File baseDir, Resource doc, String locale, TranslationsResou
File poFile = new File(localeDir, doc.getName() + ".po");
mkdirs(poFile.getParentFile());
FileWriter fWriter = new FileWriter(poFile);
- write(fWriter, doc, targetDoc);
+ write(fWriter, "UTF-8", doc, targetDoc);
}
/**
@@ -108,7 +108,7 @@ public void writePo(File baseDir, Resource doc, String locale, TranslationsResou
public void writePo(OutputStream stream, String charset, Resource doc, TranslationsResource targetDoc) throws IOException
{
OutputStreamWriter osWriter = new OutputStreamWriter(stream, charset);
- write(osWriter, doc, targetDoc);
+ write(osWriter, charset, doc, targetDoc);
}
/**
@@ -122,19 +122,19 @@ public void writePo(OutputStream stream, String charset, Resource doc, Translati
* @param targetDoc
* @throws IOException
*/
- private void write(Writer writer, Resource document, TranslationsResource targetDoc) throws IOException
+ private void write(Writer writer, String charset, Resource document, TranslationsResource targetDoc) throws IOException
{
PoHeader poHeader = document.getExtensions(true).findByType(PoHeader.class);
HeaderFields hf = new HeaderFields();
if (poHeader == null)
{
log.warn("No PO header in document named " + document.getName());
- setDefaultHeaderFields(hf);
}
else
{
copyToHeaderFields(hf, poHeader.getEntries());
}
+ setEncodingHeaderFields(hf, charset);
Message headerMessage = null;
if (targetDoc != null)
{
@@ -225,11 +225,21 @@ private static void copyTargetHeaderComments(Message headerMessage, PoTargetHead
}
}
- static void setDefaultHeaderFields(HeaderFields hf)
+ static void setEncodingHeaderFields(HeaderFields hf, String charset)
{
- hf.setValue("MIME-Version", "1.0");
- hf.setValue("Content-Type", "text/plain; charset=UTF-8");
- hf.setValue("Content-Transfer-Encoding", "8bit");
+ hf.setValue(HeaderFields.KEY_MimeVersion, "1.0");
+ hf.setValue(HeaderFields.KEY_ContentTransferEncoding, "8bit");
+
+ String ct, contentType = hf.getValue(HeaderFields.KEY_ContentType);
+ if (contentType == null)
+ {
+ ct = "text/plain; charset=" + charset;
+ }
+ else
+ {
+ ct = contentType.replaceFirst("charset=[^;]*", "charset=" + charset);
+ }
+ hf.setValue(HeaderFields.KEY_ContentType, ct);
}
static void copyToHeaderFields(HeaderFields hf, final List<HeaderEntry> entries)
0  ...a-war/src/main/java/org/zanata/util/HashUtil.java → ...-util/src/main/java/org/zanata/util/HashUtil.java
View
File renamed without changes
Please sign in to comment.
Something went wrong with that request. Please try again.