Added tools:make-emoji-tool module

settings.gradle

@@ -1,3 +1,3 @@
 include ':app'
-include ':tools'
 include ':tools:make-keyboard-text'
+include ':tools:make-emoji-keys'

tools/make-emoji-keys/build.gradle

@@ -0,0 +1,40 @@
+apply plugin: 'java'
+apply plugin: 'kotlin'
+
+version 'unspecified'
+
+jar {
+    manifest {
+        attributes["Main-Class"] = 'com/majeur/inputmethod/tools/emoji/MakeEmojiKeys'
+    }
+    from {
+        configurations.runtimeClasspath.collect {
+            it.isDirectory() ? it : zipTree(it)
+        }
+    }
+    from("src/main/ressources")
+
+}
+
+task makeEmoji(type: JavaExec, dependsOn: ['jar']) {
+    main = '-jar'
+    args jar.archiveFile.get()
+    args '-res'
+    args project.rootProject.project('app').projectDir.path + File.separator + 'src' +
+            File.separator + 'main' + File.separator + 'res'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+}
+
+
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_1_8
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/AndroidEmojiSupportFileParser.kt

@@ -0,0 +1,43 @@
+package com.majeur.inputmethod.tools.emoji
+
+
+class AndroidEmojiSupportFileParser : TextFileParser<Map<Int, Int>>() {
+
+    private val map = mutableMapOf<Int, Int>()
+    private var currentApiLevel = 0
+
+    override fun getParseResult() = map
+
+    override fun parseLine(content: String) {
+        ifStartsWith(content,
+            API_LEVEL_MARK to ::parseApiLevel,
+            UNICODE_MARK to ::parseCodePoints)
+    }
+
+    private fun parseApiLevel(content: String) {
+        currentApiLevel = content
+                .substringBefore("#")
+                .trim()
+                .toInt()
+    }
+
+    private fun parseCodePoints(content: String) {
+        val codePointsHash = content
+                .substringBefore("#")
+                .trim()
+                .split(" ")
+                .map { it
+                        .trim()
+                        .removePrefix("U+")
+                        .toInt(radix = 16) }
+                .joinToString(separator = "")
+                .hashCode()
+        map[codePointsHash] = currentApiLevel
+    }
+
+    companion object {
+
+        private const val API_LEVEL_MARK = "@"
+        private const val UNICODE_MARK = "U"
+    }
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/EmojiCategoriesResource.kt

@@ -0,0 +1,153 @@
+package com.majeur.inputmethod.tools.emoji
+
+import com.majeur.inputmethod.tools.emoji.model.EmojiData
+import com.majeur.inputmethod.tools.emoji.model.EmojiGroup
+import java.io.*
+import java.nio.charset.Charset
+import java.util.jar.JarFile
+
+class EmojiCategoriesResource(private val jarFile: JarFile) {
+
+    fun writeToAndroidRes(outDir: String?, emojiData: EmojiData, supportData: Map<Int, Int>) {
+        val template = JarUtils.getAndroidResTemplateResource(jarFile)
+        val resourceDir = template.substring(0, template.lastIndexOf('/'))
+        var ps: PrintStream? = null
+        var lnr: LineNumberReader? = null
+        try {
+            ps = if (outDir == null) {
+                System.out
+            } else {
+                val outDir = File(outDir, resourceDir)
+                val outputFile = File(outDir,
+                        ANDROID_RES_TEMPLATE.replace(".tmpl", ".xml"))
+                outDir.mkdirs()
+                println("Building android resource file into ${outputFile.absoluteFile}")
+                PrintStream(outputFile, Charset.forName("UTF-8"))
+            }
+            lnr = LineNumberReader(InputStreamReader(JarUtils.openResource(template), Charset.forName("UTF-8")))
+            inflateTemplate(lnr, ps!!, emojiData, supportData)
+        } catch (e: IOException) {
+            throw RuntimeException(e)
+        } finally {
+            JarUtils.close(lnr)
+            JarUtils.close(ps)
+        }
+    }
+
+    @Throws(IOException::class)
+    private fun inflateTemplate(reader: LineNumberReader, out: PrintStream,
+                                emojis: EmojiData, supportData: Map<Int, Int>) {
+        reader.lines().forEach {
+            when {
+                it.contains(MARK_UNICODE_VER) ->
+                    out.println(it.replace(MARK_UNICODE_VER, emojis.unicodeVersion))
+                it.contains(MARK_API_LEVEL) ->
+                    out.println(it.replace(MARK_API_LEVEL, supportData.values.maxOrNull().toString()))
+                it.contains(MARK_SMILEYS_AND_EMOTION) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.SMILEYS_AND_EMOTION)
+                it.contains(MARK_PEOPLE_AND_BODY) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.PEOPLE_AND_BODY)
+                it.contains(MARK_ANIMALS_AND_NATURE) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.ANIMALS_AND_NATURE)
+                it.contains(MARK_FOOD_AND_DRINK) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.FOOD_AND_DRINK)
+                it.contains(MARK_TRAVEL_AND_PLACES) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.TRAVEL_AND_PLACES)
+                it.contains(MARK_ACTIVITIES) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.ACTIVITIES)
+                it.contains(MARK_OBJECTS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.OBJECTS)
+                it.contains(MARK_SYMBOLS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.SYMBOLS)
+                it.contains(MARK_FLAGS) ->
+                    dumpEmojiSpecs(out, emojis, supportData,EmojiGroup.FLAGS)
+                it.contains(MARK_PEOPLE_AND_BODY_MORE) ->
+                    dumpEmojiSpecsVariant(out, emojis, supportData,EmojiGroup.PEOPLE_AND_BODY)
+                else -> out.println(it)
+            }
+        }
+    }
+
+    private fun dumpEmojiSpecs(out: PrintStream, emojiData: EmojiData, supportData: Map<Int, Int>,
+                               group: EmojiGroup) {
+        emojiData[group].forEach { emoji ->
+            val minApi = getMinApi(emoji.codes, supportData)
+            if (minApi < 0) {
+                // We have no clue of which android version supports this emoji,
+                // so we ignore it.
+                printCompatNotFound(emoji.codes)
+                return@forEach
+            }
+            val text = makeEmojiKey(emoji.codes, minApi)
+            out.println("        <item>$text</item>")
+        }
+    }
+
+    private fun dumpEmojiSpecsVariant(out: PrintStream, emojiData: EmojiData, supportData: Map<Int, Int>,
+                               group: EmojiGroup) {
+        emojiData[group].forEach { baseEmoji ->
+            val minApi = getMinApi(baseEmoji.codes, supportData)
+            if (minApi < 0) {
+                // Same thing, we already encountered it when dumping base emoji,
+                // ignoring this one silently.
+                return@forEach
+            }
+
+            val text = baseEmoji.variants.filter { emoji ->
+                if (getMinApi(emoji.codes, supportData) < 0) {
+                    // Again
+                    printCompatNotFound(emoji.codes)
+                    return@filter false
+                }
+                true
+            }.map { emoji ->
+                // Not very efficient, minApi is accessed twice,
+                // but hey, we are making tooling here
+                makeEmojiKey(emoji.codes, getMinApi(emoji.codes, supportData))
+            }.filter { key ->
+                key.isNotBlank()
+            }.joinToString(separator = ";")
+
+            if (text.isNotBlank()) out.println("        <item>$text</item>")
+            else out.println("        <item/>")
+        }
+    }
+
+    private fun makeEmojiKey(codes: IntArray, minApi: Int): String {
+        val cps = codes
+                .joinToString(separator = ",") {
+                    it.toString(radix = 16)
+                            .uppercase()
+                }
+        return if (minApi > 19) "$cps||$minApi" else cps
+    }
+
+    private fun getMinApi(codes: IntArray, supportData: Map<Int, Int>): Int {
+        val hash = codes
+                .joinToString(separator = "")
+                .hashCode()
+        return supportData[hash] ?: -1
+    }
+
+    private fun printCompatNotFound(codes: IntArray) {
+        val formattedCps = codes.joinToString(" ") { "U+" + it.toString(radix = 16).uppercase() }
+        println(" - No android compatibility found for emoji $formattedCps, ignoring...")
+    }
+
+    companion object {
+        private const val ANDROID_RES_TEMPLATE = "emoji-categories.tmpl"
+        private const val MARK_UNICODE_VER = "@UNICODE_VERSION@"
+        private const val MARK_API_LEVEL = "@ANDROID_API_LEVEL@"
+        private const val MARK_SMILEYS_AND_EMOTION = "@SMILEYS_AND_EMOTION@"
+        private const val MARK_PEOPLE_AND_BODY = "@PEOPLE_AND_BODY@"
+        private const val MARK_PEOPLE_AND_BODY_MORE = "@PEOPLE_AND_BODY MORE@"
+        private const val MARK_ANIMALS_AND_NATURE = "@ANIMALS_AND_NATURE@"
+        private const val MARK_FOOD_AND_DRINK = "@FOOD_AND_DRINKS@"
+        private const val MARK_TRAVEL_AND_PLACES = "@TRAVEL_AND_PLACES@"
+        private const val MARK_ACTIVITIES = "@ACTIVITIES@"
+        private const val MARK_OBJECTS = "@OBJECTS@"
+        private const val MARK_SYMBOLS = "@SYMBOLS@"
+        private const val MARK_FLAGS = "@FLAGS@"
+    }
+
+}

tools/make-emoji-keys/src/main/kotlin/com/majeur/inputmethod/tools/emoji/EmojiUCDTestFileParser.kt

@@ -0,0 +1,88 @@
+package com.majeur.inputmethod.tools.emoji
+
+import com.majeur.inputmethod.tools.emoji.model.EmojiData
+import com.majeur.inputmethod.tools.emoji.model.EmojiGroup
+
+class EmojiUCDTestFileParser: TextFileParser<EmojiData>() {
+
+    private var count = 0
+    private var emojiData = EmojiData()
+
+    private var currentGroup = EmojiGroup.SMILEYS_AND_EMOTION
+
+    override fun getParseResult() = emojiData
+
+    override fun parseLine(content: String) {
+        ifStartsWith(content,
+            "#" to ::parseComment,
+            "" to ::parseEmojiSpec
+        )
+    }
+
+    private fun parseComment(content: String) {
+        ifStartsWith(content,
+            PROP_DATE to { emojiData.dataDate = it},
+            PROP_UNICODE_VER to {
+                emojiData.unicodeVersion = it
+                println("Parsing emoji table from Unicode $it")
+            },
+            PROP_GROUP to ::parseGroup,
+            PROP_SUBGROUP to { },
+            "${currentGroup.rawName} subtotal:" to ::parseGroupSubtotal,
+            EOF to { println("Parsed a total of $count emojis") }
+        )
+    }
+
+    private fun parseGroup(content: String) {
+        currentGroup = EmojiGroup.get(content)
+    }
+
+    private fun parseGroupSubtotal(content: String) {
+        if (content.contains("w/o modifiers")) return
+        val expected = content.toInt()
+        val count = emojiData.emojiGroupCount(currentGroup)
+        println(" - $count/$expected emojis for group ${currentGroup.rawName}")
+    }
+
+    private fun parseEmojiSpec(content: String) {
+        if (content.isEmpty()) return
+
+        val codePoints = content
+                .substringBefore(';')
+                .trim()
+        val status = content
+                .substringAfter(';')
+                .substringBefore('#')
+                .trim()
+        val extras = content.substringAfter('#')
+
+        if (status != "fully-qualified") return
+
+        val rawVersion = EMOJI_VERSION_REGEX.find(extras)?.value ?: "O.0"
+        val version = rawVersion.toFloat()
+        val name = extras
+                .substringAfter(rawVersion)
+                .trim()
+
+        val cps = codePoints
+                .split(" ")
+                .map { it.toInt(radix = 16) }
+                .toIntArray()
+
+        emojiData.insertEmoji(currentGroup, cps, version, name)
+        count++
+    }
+
+    companion object {
+
+        private const val PROP_UNICODE_VER = "Version:"
+        private const val PROP_DATE = "Date:"
+        private const val PROP_GROUP = "group:"
+        private const val PROP_SUBGROUP = "subgroup:"
+        private const val EOF = "EOF"
+
+        private val EMOJI_VERSION_REGEX = "[0-9]*[.]?[0-9]+".toRegex()
+    }
+
+
+}