Compress dictionaries and allow users to add more

app/build.gradle

@@ -36,7 +36,8 @@ android {
 
     ndkVersion '21.3.6528147'
     androidResources {
-        noCompress 'dict'
+        noCompress 'main.dict'
+        noCompress 'empty.dict'
     }
 }
 

app/src/main/java/org/dslul/openboard/inputmethod/latin/BinaryDictionaryGetter.java

@@ -21,6 +21,7 @@
 import android.content.res.AssetFileDescriptor;
 import android.util.Log;
 
+import org.dslul.openboard.inputmethod.latin.common.FileUtils;
 import org.dslul.openboard.inputmethod.latin.common.LocaleUtils;
 import org.dslul.openboard.inputmethod.latin.define.DecoderSpecificConstants;
 import org.dslul.openboard.inputmethod.latin.makedict.DictionaryHeader;
@@ -62,6 +63,8 @@ final public class BinaryDictionaryGetter {
     public static final String MAIN_DICTIONARY_CATEGORY = "main";
     public static final String ID_CATEGORY_SEPARATOR = ":";
 
+    public static final String ASSETS_DICTIONARY_FOLDER = "dicts";
+
     // The key considered to read the version attribute in a dictionary file.
     private static String VERSION_KEY = "version";
 
@@ -265,14 +268,83 @@ public static ArrayList<AssetFileAddress> getDictionaryFiles(final Locale locale
         }
 
         if (!foundMainDict && dictPackSettings.isWordListActive(mainDictId)) {
-            final int fallbackResId =
-                    DictionaryInfoUtils.getMainDictionaryResourceId(context.getResources(), locale);
-            final AssetFileAddress fallbackAsset = loadFallbackResource(context, fallbackResId);
+            final File dict = loadDictionaryFromAssets(locale.toString(), context);
+            final AssetFileAddress fallbackAsset;
+            if (dict == null) {
+                // fall back to the old way (maybe remove? will not work if files are compressed)
+                final int fallbackResId =
+                        DictionaryInfoUtils.getMainDictionaryResourceId(context.getResources(), locale);
+                fallbackAsset = loadFallbackResource(context, fallbackResId);
+            } else {
+                fallbackAsset = AssetFileAddress.makeFromFileName(dict.getPath());
+            }
             if (null != fallbackAsset) {
                 fileList.add(fallbackAsset);
             }
         }
 
         return fileList;
     }
+
+    /**
+     * Returns the best matching main dictionary from assets.
+     *
+     * Actually copies the dictionary to cache folder, and then returns that file. This allows
+     * the dictionaries to be stored in a compressed way, reducing APK size.
+     * On next load, the dictionary in cache folder is found by getCachedWordLists
+     *
+     * Returns null on IO errors or if no matching dictionary is found
+     */
+    public static File loadDictionaryFromAssets(final String locale, final Context context) {
+        final String[] dictionaryList;
+        try {
+            dictionaryList = context.getAssets().list(ASSETS_DICTIONARY_FOLDER);
+        } catch (IOException e) {
+            return null;
+        }
+        if (null == dictionaryList) return null;
+        String bestMatchName = null;
+        int bestMatchLevel = 0;
+        for (String dictionary : dictionaryList) {
+            final String dictLocale =
+                    extractLocaleFromAssetsDictionaryFile(dictionary);
+            if (dictLocale == null) continue;
+            final int matchLevel = LocaleUtils.getMatchLevel(dictLocale, locale);
+            if (LocaleUtils.isMatch(matchLevel) && matchLevel > bestMatchLevel) {
+                bestMatchName = dictionary;
+            }
+        }
+        if (bestMatchName == null) return null;
+
+        // we have a match, now copy contents of the dictionary to "cached" word lists folder
+        File dictFile = new File(DictionaryInfoUtils.getCacheDirectoryForLocale(bestMatchName, context) +
+                File.separator + DictionaryInfoUtils.MAIN_DICTIONARY_INTERNAL_FILE_NAME);
+        try {
+            FileUtils.copyStreamToNewFile(
+                    context.getAssets().open(ASSETS_DICTIONARY_FOLDER + File.separator + bestMatchName),
+                    dictFile);
+            return dictFile;
+        } catch (IOException e) {
+            Log.e(TAG, "exception while looking for locale " + locale, e);
+            return null;
+        }
+    }
+
+    /**
+     * Returns the locale for a dictionary file name stored in assets.
+     *
+     * Assumes file name main_[locale].dict
+     *
+     * Returns the locale, or null if file name does not match the pattern
+     */
+    private static String extractLocaleFromAssetsDictionaryFile(final String dictionaryFileName) {
+        if (dictionaryFileName.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY)
+                && dictionaryFileName.endsWith(".dict")) {
+            return dictionaryFileName.substring(
+                    BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.length() + 1,
+                    dictionaryFileName.lastIndexOf('.')
+            );
+        }
+        return null;
+    }
 }

app/src/main/java/org/dslul/openboard/inputmethod/latin/common/FileUtils.java

@@ -17,7 +17,10 @@
 package org.dslul.openboard.inputmethod.latin.common;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
 
 /**
  * A simple class to help with removing directories recursively.
@@ -58,4 +61,19 @@ public static boolean renameTo(final File fromFile, final File toFile) {
         toFile.delete();
         return fromFile.renameTo(toFile);
     }
+
+    public static void copyStreamToNewFile(InputStream in, File outfile) throws IOException {
+        File parentFile = outfile.getParentFile();
+        if (parentFile == null || (!parentFile.exists() && !parentFile.mkdirs())) {
+            throw new IOException("could not create parent folder");
+        }
+        FileOutputStream out = new FileOutputStream(outfile);
+        byte[] buf = new byte[1024];
+        int len;
+        while ((len = in.read(buf)) > 0) {
+            out.write(buf, 0, len);
+        }
+        out.flush();
+    }
+
 }

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/CorrectionSettingsFragment.java

@@ -16,24 +16,34 @@
 
 package org.dslul.openboard.inputmethod.latin.settings;
 
-import android.Manifest;
 import android.app.Activity;
+import android.app.AlertDialog;
 import android.content.Context;
+import android.content.DialogInterface;
 import android.content.Intent;
 import android.content.SharedPreferences;
 import android.content.pm.PackageManager;
 import android.content.pm.ResolveInfo;
+import android.net.Uri;
 import android.os.Bundle;
 import android.preference.Preference;
-import android.preference.SwitchPreference;
-import android.text.TextUtils;
+import android.text.Html;
+import android.text.Spanned;
+import android.text.method.LinkMovementMethod;
+import android.widget.TextView;
+import android.widget.Toast;
 
+import org.dslul.openboard.inputmethod.dictionarypack.DictionaryPackConstants;
 import org.dslul.openboard.inputmethod.latin.R;
-import org.dslul.openboard.inputmethod.latin.permissions.PermissionsManager;
-import org.dslul.openboard.inputmethod.latin.permissions.PermissionsUtil;
+import org.dslul.openboard.inputmethod.latin.common.FileUtils;
+import org.dslul.openboard.inputmethod.latin.makedict.DictionaryHeader;
 import org.dslul.openboard.inputmethod.latin.userdictionary.UserDictionaryList;
 import org.dslul.openboard.inputmethod.latin.userdictionary.UserDictionarySettings;
+import org.dslul.openboard.inputmethod.latin.utils.DialogUtils;
+import org.dslul.openboard.inputmethod.latin.utils.DictionaryInfoUtils;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.TreeSet;
 
 /**
@@ -55,6 +65,8 @@ public final class CorrectionSettingsFragment extends SubScreenFragment
     private static final boolean DBG_USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS = false;
     private static final boolean USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS =
             DBG_USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS;
+    private static final int DICTIONARY_REQUEST_CODE = 96834;
+    private static final String DICTIONARY_URL = "https://github.com/openboard-team/openboard/"; // TODO: update once it exists
 
     @Override
     public void onCreate(final Bundle icicle) {
@@ -73,6 +85,19 @@ public void onCreate(final Bundle icicle) {
         if (ri == null) {
             overwriteUserDictionaryPreference(editPersonalDictionary);
         }
+
+        // Ideally this would go to a preference screen where extra dictionaries can be managed
+        // so user can check which dictionaries exists (internal and added), and also delete them.
+        // But for now just adding new ones and replacing is ok.
+        final Preference addDictionary = findPreference(Settings.PREF_ADD_DICTIONARY);
+        if (addDictionary != null)
+            addDictionary.setOnPreferenceClickListener(new Preference.OnPreferenceClickListener() {
+                @Override
+                public boolean onPreferenceClick(Preference preference) {
+                    showAddDictionaryDialog();
+                    return true;
+                }
+            });
     }
 
     private void overwriteUserDictionaryPreference(final Preference userDictionaryPreference) {
@@ -99,4 +124,111 @@ private void overwriteUserDictionaryPreference(final Preference userDictionaryPr
         }
     }
 
+    private void showAddDictionaryDialog() {
+        final String link = "<a href='" + DICTIONARY_URL + "'>" +
+                getResources().getString(R.string.dictionary_selection_link_text) + "</a>";
+        final Spanned message = Html.fromHtml(getResources().getString(R.string.dictionary_selection_message, link));
+        final AlertDialog dialog = new AlertDialog.Builder(
+                DialogUtils.getPlatformDialogThemeContext(getActivity()))
+                .setTitle(R.string.dictionary_selection_title)
+                .setMessage(message)
+                .setNegativeButton(R.string.cancel, null)
+                .setPositiveButton(R.string.dictionary_selection_load_file, new DialogInterface.OnClickListener() {
+                    @Override
+                    public void onClick(DialogInterface dialog, int which) {
+                        Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT)
+                                .addCategory(Intent.CATEGORY_OPENABLE)
+                                .setType("application/octet-stream");
+                        startActivityForResult(intent, DICTIONARY_REQUEST_CODE);
+                    }
+                })
+                .create();
+        dialog.show();
+        // make links in the HTML text work
+        ((TextView) dialog.findViewById(android.R.id.message))
+                .setMovementMethod(LinkMovementMethod.getInstance());
+    }
+
+    private void onDictionaryFileSelected(int resultCode, Intent resultData) {
+        if (resultCode != Activity.RESULT_OK || resultData == null) {
+            onDictionaryLoadingError(R.string.dictionary_selection_error);
+            return;
+        }
+
+        final Uri uri = resultData.getData();
+        if (uri == null) {
+            onDictionaryLoadingError(R.string.dictionary_selection_error);
+            return;
+        }
+
+        final File cachedDictionaryFile = new File(getActivity().getCacheDir().getPath() + File.separator + "temp_dict");
+        try {
+            FileUtils.copyStreamToNewFile(
+                    getActivity().getContentResolver().openInputStream(uri),
+                    cachedDictionaryFile);
+        } catch (IOException e) {
+            onDictionaryLoadingError(R.string.dictionary_selection_error);
+            return;
+        }
+
+        final DictionaryHeader newHeader = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length());
+        if (newHeader == null) {
+            cachedDictionaryFile.delete();
+            onDictionaryLoadingError(R.string.dictionary_selection_file_error);
+            return;
+        }
+
+        final String dictFolder =
+                DictionaryInfoUtils.getCacheDirectoryForLocale(newHeader.mLocaleString, getActivity());
+        final File dictFile = new File(dictFolder + File.separator + DictionaryInfoUtils.MAIN_DICTIONARY_USER_FILE_NAME);
+        if (dictFile.exists()) {
+            final DictionaryHeader oldHeader =
+                    DictionaryInfoUtils.getDictionaryFileHeaderOrNull(dictFile, 0, dictFile.length());
+            if (oldHeader != null
+                    && Integer.parseInt(oldHeader.mVersionString) > Integer.parseInt(newHeader.mVersionString)
+                    && !shouldReplaceExistingUserDictionary()) {
+                cachedDictionaryFile.delete();
+                return;
+            }
+        }
+
+        if (!cachedDictionaryFile.renameTo(dictFile)) {
+            cachedDictionaryFile.delete();
+            onDictionaryLoadingError(R.string.dictionary_selection_error);
+            return;
+        }
+
+        // success, now remove internal dictionary file if it exists
+        final File internalDictFile = new File(dictFolder + File.separator +
+                DictionaryInfoUtils.MAIN_DICTIONARY_INTERNAL_FILE_NAME);
+        if (internalDictFile.exists())
+            internalDictFile.delete();
+
+        // inform user about success
+        final String successMessageForLocale = getResources()
+                .getString(R.string.dictionary_selection_load_success, newHeader.mLocaleString);
+        Toast.makeText(getActivity(), successMessageForLocale, Toast.LENGTH_SHORT).show();
+
+        // inform LatinIME about new dictionary
+        final Intent newDictBroadcast = new Intent(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION);
+        getActivity().sendBroadcast(newDictBroadcast);
+    }
+
+    private void onDictionaryLoadingError(int resId) {
+        // show error message... maybe better as dialog so user definitely notices?
+        Toast.makeText(getActivity(), resId, Toast.LENGTH_LONG).show();
+    }
+
+    private boolean shouldReplaceExistingUserDictionary() {
+        // TODO: show dialog, ask user whether existing file should be replaced
+        // return true if yes, no otherwise (set .setCancelable(false) to avoid dismissing without the buttons!)
+        return true;
+    }
+
+    @Override
+    public void onActivityResult(int requestCode, int resultCode, Intent resultData) {
+        if (requestCode == DICTIONARY_REQUEST_CODE)
+            onDictionaryFileSelected(resultCode, resultData);
+    }
+
 }

app/src/main/java/org/dslul/openboard/inputmethod/latin/settings/Settings.java

@@ -65,6 +65,7 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang
     public static final String PREF_VOICE_INPUT_KEY = "pref_voice_input_key";
     public static final String PREF_CLIPBOARD_CLIPBOARD_KEY = "pref_clipboard_clipboard_key";
     public static final String PREF_EDIT_PERSONAL_DICTIONARY = "edit_personal_dictionary";
+    public static final String PREF_ADD_DICTIONARY = "add_dictionary";
     // PREF_AUTO_CORRECTION_THRESHOLD_OBSOLETE is obsolete. Use PREF_AUTO_CORRECTION instead.
     public static final String PREF_AUTO_CORRECTION_THRESHOLD_OBSOLETE =
             "auto_correction_threshold";

app/src/main/java/org/dslul/openboard/inputmethod/latin/utils/DictionaryInfoUtils.java

@@ -54,6 +54,8 @@ public class DictionaryInfoUtils {
     public static final String RESOURCE_PACKAGE_NAME = R.class.getPackage().getName();
     private static final String DEFAULT_MAIN_DICT = "main";
     private static final String MAIN_DICT_PREFIX = "main_";
+    public static final String MAIN_DICTIONARY_INTERNAL_FILE_NAME = DEFAULT_MAIN_DICT + ".dict";
+    public static final String MAIN_DICTIONARY_USER_FILE_NAME = MAIN_DICT_PREFIX + "user.dict";
     private static final String DECODER_DICT_SUFFIX = DecoderSpecificConstants.DECODER_DICT_SUFFIX;
     // 6 digits - unicode is limited to 21 bits
     private static final int MAX_HEX_DIGITS_FOR_CODEPOINT = 6;
@@ -151,7 +153,7 @@ public static String replaceFileNameDangerousCharacters(final String name) {
     /**
      * Helper method to get the top level cache directory.
      */
-    private static String getWordListCacheDirectory(final Context context) {
+    public static String getWordListCacheDirectory(final Context context) {
         return context.getFilesDir() + File.separator + "dicts";
     }
 
@@ -242,7 +244,7 @@ public static boolean isMainWordListId(final String id) {
         // An id is supposed to be in format category:locale, so splitting on the separator
         // should yield a 2-elements array
         if (2 != idArray.length) {
-            return false;
+            return id.startsWith(BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY);
         }
         return BinaryDictionaryGetter.MAIN_DICTIONARY_CATEGORY.equals(idArray[0]);
     }

app/src/main/res/values/strings.xml

@@ -461,6 +461,20 @@ disposition rather than other common dispositions for Latin languages. [CHAR LIM
     <string name="dictionary_settings_title">Add-on dictionaries</string>
     <!-- Title for the prompt dialog which informs the user that a dictionary is available for the current language and asks to decide whether to download it over 3g -->
     <string name="dictionary_settings_summary">Settings for dictionaries</string>
+    <!-- Title for the user dictionary selection dialog -->
+    <string name="dictionary_selection_title">"Choose dictionary file"</string>
+    <!-- Message for the user dictionary selection dialog. This string will be interpreted as HTML -->
+    <string name="dictionary_selection_message">"Select a dictionary to replace the main dictionary of the same locale. Dictionaries can be downloaded at %s."</string>
+    <!-- Title of the link to the download page inserted into selection message -->
+    <string name="dictionary_selection_link_text">"the project repository"</string>
+    <!-- Button text for dictionary file selection -->
+    <string name="dictionary_selection_load_file">"Load dictionary"</string>
+    <!-- Toast text shown when dictionary file was added successfully -->
+    <string name="dictionary_selection_load_success">"Dictionary for locale \"%s\" added"</string>
+    <!-- Text shown when dictionary file could not be read -->
+    <string name="dictionary_selection_file_error">"Error: Selected file is not a valid dictionary file"</string>
+    <!-- Text shown on other errors when loading dictionary file -->
+    <string name="dictionary_selection_error">"Error loading dictionary file"</string>
     <!-- Name of the user dictionaries settings category -->
     <string name="user_dictionaries">User dictionaries</string>
     <!-- Name for the "user dictionary" preference item when there is only one -->

app/src/main/res/xml/prefs_screen_correction.xml

@@ -24,6 +24,10 @@
         <intent android:action="android.settings.USER_DICTIONARY_SETTINGS" />
     </PreferenceScreen>
 
+    <Preference
+        android:key="add_dictionary"
+        android:title="@string/configure_dictionaries_title" />
+
     <PreferenceCategory
         android:title="@string/settings_category_correction">