diff --git a/CodenameOneDesigner/src/com/codename1/designer/L10nEditor.java b/CodenameOneDesigner/src/com/codename1/designer/L10nEditor.java index 7b72526e82..a7e23395e7 100644 --- a/CodenameOneDesigner/src/com/codename1/designer/L10nEditor.java +++ b/CodenameOneDesigner/src/com/codename1/designer/L10nEditor.java @@ -26,6 +26,7 @@ import com.codename1.designer.ResourceEditorView; import com.codename1.io.CSVParser; +import com.codename1.tools.resourcebuilder.PropertiesUtil; import com.codename1.ui.plaf.Accessor; import com.codename1.ui.resource.util.SwingRenderer; import com.codename1.ui.util.EditableResources; @@ -987,7 +988,7 @@ public void skippedEntity(String name) throws SAXException { } } else { Properties prop = new Properties(); - prop.load(f); + PropertiesUtil.loadUtf8WithFallback(files[0], prop); for (Object key : prop.keySet()) { res.setLocaleProperty(localeName, locale, (String)key, prop.getProperty((String)key)); } diff --git a/CodenameOneDesigner/src/com/codename1/designer/css/CN1CSSCLI.java b/CodenameOneDesigner/src/com/codename1/designer/css/CN1CSSCLI.java index 811e5b2e16..fbb2f2e757 100644 --- a/CodenameOneDesigner/src/com/codename1/designer/css/CN1CSSCLI.java +++ b/CodenameOneDesigner/src/com/codename1/designer/css/CN1CSSCLI.java @@ -30,6 +30,7 @@ import com.codename1.impl.javase.CN1Bootstrap; import com.codename1.io.Log; import com.codename1.io.Util; +import com.codename1.tools.resourcebuilder.PropertiesUtil; import com.codename1.ui.BrowserComponent; import com.codename1.ui.CN; import com.codename1.ui.Component; @@ -1106,9 +1107,7 @@ private static Map>> loadLocalizationBun return; } Properties props = new Properties(); - try (InputStream is = Files.newInputStream(p)) { - props.load(is); - } + PropertiesUtil.loadUtf8WithFallback(p.toFile(), props); Map> baseBundles = bundles.computeIfAbsent(baseName, k -> new LinkedHashMap<>()); Map translations = new LinkedHashMap<>(); for (Map.Entry entry : props.entrySet()) { diff --git a/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/L10NTask.java b/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/L10NTask.java index 561fb4890e..e11ac3827e 100644 --- a/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/L10NTask.java +++ b/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/L10NTask.java @@ -27,9 +27,7 @@ import com.codename1.ui.util.EditableResources; import java.io.DataOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Hashtable; import java.util.List; @@ -67,9 +65,7 @@ public void addToResources(EditableResources e) throws IOException { throw new BuildException("Both name and file attributes of the locale task are required attributes"); } Properties p = new Properties(); - InputStream i = new FileInputStream(l.getFile()); - p.load(i); - i.close(); + PropertiesUtil.loadUtf8WithFallback(l.getFile(), p); e.addLocale(getName(), l.getName()); for(Object key : p.keySet()) { e.setLocaleProperty(getName(), l.getName(), (String)key, p.getProperty((String)key)); diff --git a/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/PropertiesUtil.java b/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/PropertiesUtil.java new file mode 100644 index 0000000000..8301ec87fc --- /dev/null +++ b/CodenameOneDesigner/src/com/codename1/tools/resourcebuilder/PropertiesUtil.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores + * CA 94065 USA or visit www.oracle.com if you need additional information or + * have any questions. + */ + +package com.codename1.tools.resourcebuilder; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.MalformedInputException; +import java.nio.charset.StandardCharsets; +import java.nio.charset.UnmappableCharacterException; +import java.nio.file.Files; +import java.util.Properties; + +/** + * Loader for {@code .properties} files that supports both UTF-8 and the legacy + * ISO-8859-1 / {@code native2ascii} encoding. + * + *

This mirrors the behavior of {@code java.util.PropertyResourceBundle} as + * updated in JDK 9 (JEP 226): the file is read first as UTF-8, and if + * the bytes are not valid UTF-8 the loader falls back to ISO-8859-1 so existing + * files produced by {@code native2ascii} (or otherwise stored in Latin-1) keep + * working. Standard {@code \\uXXXX} escapes recognized by + * {@link Properties#load(Reader)} are honored in either mode.

+ */ +public final class PropertiesUtil { + + private PropertiesUtil() {} + + /** + * Loads {@code file} into {@code props}, preferring UTF-8 and falling back + * to ISO-8859-1 if the file is not valid UTF-8. + */ + public static void loadUtf8WithFallback(File file, Properties props) throws IOException { + byte[] data = Files.readAllBytes(file.toPath()); + loadUtf8WithFallback(data, props); + } + + /** + * Loads the contents of {@code in} into {@code props}, preferring UTF-8 and + * falling back to ISO-8859-1 if the bytes are not valid UTF-8. The stream + * is fully read but not closed. + */ + public static void loadUtf8WithFallback(InputStream in, Properties props) throws IOException { + loadUtf8WithFallback(readAll(in), props); + } + + private static void loadUtf8WithFallback(byte[] data, Properties props) throws IOException { + Properties decoded = new Properties(); + try (Reader r = new InputStreamReader(new ByteArrayInputStream(data), + StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT))) { + decoded.load(r); + } catch (MalformedInputException | UnmappableCharacterException ex) { + decoded = new Properties(); + try (Reader r = new InputStreamReader(new ByteArrayInputStream(data), + StandardCharsets.ISO_8859_1)) { + decoded.load(r); + } + } + for (String name : decoded.stringPropertyNames()) { + props.setProperty(name, decoded.getProperty(name)); + } + } + + private static byte[] readAll(InputStream in) throws IOException { + java.io.ByteArrayOutputStream buf = new java.io.ByteArrayOutputStream(); + byte[] tmp = new byte[8192]; + int n; + while ((n = in.read(tmp)) > 0) { + buf.write(tmp, 0, n); + } + return buf.toByteArray(); + } +} diff --git a/CodenameOneDesigner/test/com/codename1/designer/css/CSSLocalizationTest.java b/CodenameOneDesigner/test/com/codename1/designer/css/CSSLocalizationTest.java index e851078c4f..164b86cc59 100644 --- a/CodenameOneDesigner/test/com/codename1/designer/css/CSSLocalizationTest.java +++ b/CodenameOneDesigner/test/com/codename1/designer/css/CSSLocalizationTest.java @@ -25,6 +25,65 @@ public class CSSLocalizationTest { public static void main(String[] args) throws Exception { testLoadLocalizationBundles(); testApplyLocalizationBundles(); + testLoadLocalizationBundlesUtf8(); + testLoadLocalizationBundlesLatin1Fallback(); + testLoadLocalizationBundlesUnicodeEscape(); + } + + private static void testLoadLocalizationBundlesUtf8() throws Exception { + Path tempDir = Files.createTempDirectory("cn1-css-localization-utf8"); + try { + Path localizationRoot = Files.createDirectory(tempDir.resolve("l10n")); + // The exact string from issue #4883 — UTF-8 encoded Italian with accented à. + String value = "Non ci sono ancora attività. Usa il pulsante flottante per aggiungere la prima routine."; + Files.write(localizationRoot.resolve("Bundle_it.properties"), + ("home.empty=" + value + System.lineSeparator()).getBytes(StandardCharsets.UTF_8)); + + Map>> bundles = loadLocalizationBundles(localizationRoot.toFile()); + Map> bundle = bundles.get("Bundle"); + assertTrue(bundle != null, "Bundle should be detected"); + assertEquals(bundle.get("it"), stringMap("home.empty", value), + "UTF-8 encoded accented characters should round-trip"); + } finally { + deleteRecursively(tempDir); + } + } + + private static void testLoadLocalizationBundlesLatin1Fallback() throws Exception { + Path tempDir = Files.createTempDirectory("cn1-css-localization-latin1"); + try { + Path localizationRoot = Files.createDirectory(tempDir.resolve("l10n")); + // Pre-Java-9 native2ascii-style file: ISO-8859-1 with a literal accented byte. + String value = "café"; + Files.write(localizationRoot.resolve("Bundle_fr.properties"), + ("greeting=" + value + System.lineSeparator()).getBytes(StandardCharsets.ISO_8859_1)); + + Map>> bundles = loadLocalizationBundles(localizationRoot.toFile()); + Map> bundle = bundles.get("Bundle"); + assertTrue(bundle != null, "Bundle should be detected"); + assertEquals(bundle.get("fr"), stringMap("greeting", value), + "Latin-1 (legacy) bytes should fall back from UTF-8 decoding"); + } finally { + deleteRecursively(tempDir); + } + } + + private static void testLoadLocalizationBundlesUnicodeEscape() throws Exception { + Path tempDir = Files.createTempDirectory("cn1-css-localization-uesc"); + try { + Path localizationRoot = Files.createDirectory(tempDir.resolve("l10n")); + // native2ascii output: pure ASCII with backslash-u escapes. + Files.write(localizationRoot.resolve("Bundle_it.properties"), + ("home.empty=attivit\\u00e0" + System.lineSeparator()).getBytes(StandardCharsets.US_ASCII)); + + Map>> bundles = loadLocalizationBundles(localizationRoot.toFile()); + Map> bundle = bundles.get("Bundle"); + assertTrue(bundle != null, "Bundle should be detected"); + assertEquals(bundle.get("it"), stringMap("home.empty", "attività"), + "\\uXXXX escapes should still be decoded"); + } finally { + deleteRecursively(tempDir); + } } private static void testLoadLocalizationBundles() throws Exception {