src: Distinguish Arabic XKB and Keypad XKB options

Shift-NumLock keys provides U+FEF9 keysym in keypad:pointerkeys XKB
options and Shift-t keys provides U+100FEF9 keysym with Arabic layout
and IBus internal APIs handles guint16 and cannot recognize the
difference.
Now the APIs are updated to handle guint.
Also libX11 compose file provides the 16bit key sequence only and
IBus now provides a pad U+1000000 internally.

Seems the pad is added when <Uxxxx> format is used in the compose file.
Once the string sequence is converted to guint sequence, IBus
hardly to find the padded keysym and IBus uses ibus_keyval_name()
if the value has the keysym names or Unicode point but some code
points have both keysymnames and Unicode and now IBus prohibits
"Pointer_*" keysym names in compose file

Add src/tests/ibus-compose.basic test case.

To test U0227, Set SGS layout(Samogitian) in xkb/symbols/lt
and set XKB option lv3:bksl_switch and Press backslash-a keys.

BUG=#2495

src/ibuscomposetable.c

@@ -478,7 +478,8 @@ ibus_compose_list_parse_file (const char *compose_file,
             }
         }
         if (include && *include) {
-            GList *rest = ibus_compose_list_parse_file (include,
+            GList *rest = ibus_compose_list_parse_file (
+                    include,
                     max_compose_len);
             if (rest) {
                 compose_list = g_list_concat (compose_list, rest);
@@ -498,12 +499,12 @@ ibus_compose_list_check_duplicated (GList  *compose_list,
                                     GSList *compose_tables)
 {
     GList *list;
-    static guint16 *keysyms;
+    static guint *keysyms;
     GList *removed_list = NULL;
     IBusComposeData *compose_data;
     GString *output = g_string_new ("");
 
-    keysyms = g_new (guint16, max_compose_len + 1);
+    keysyms = g_new (guint, max_compose_len + 1);
 
     for (list = compose_list; list != NULL; list = list->next) {
         int i;
@@ -523,7 +524,7 @@ ibus_compose_list_check_duplicated (GList  *compose_list,
 
         for (i = 0; i < max_compose_len + 1; i++) {
             gunichar codepoint = compose_data->sequence[i];
-            keysyms[i] = (guint16) codepoint;
+            keysyms[i] = (guint)codepoint;
 
             if (codepoint == 0)
                 break;
@@ -600,6 +601,34 @@ ibus_compose_list_check_duplicated (GList  *compose_list,
 }
 
 
+/*
+ * Actual typed keysyms have a flag against the definition.
+ * https://gitlab.freedesktop.org/xorg/lib/libx11/-/blob/master/nls/en_US.UTF-8/Compose.pre#L4559
+ */
+guint
+ibus_compose_key_flag (guint key)
+{
+    const char *name;
+    if (key <= 0xff)
+        return 0;
+    name = ibus_keyval_name (key);
+    /* If name is null, the key sequence is expressed as "<Uxxxx>" format in
+     * the Compose file and the typed keysym has the flag.
+     */
+    if (!name)
+        return 0x1000000;
+    /* "<Pointer_EnableKeys>" is not described in the Compose file but <UFEF9>
+     * in the file.
+     */
+    if (*name == 'P' && *(name + 1) == 'o' && *(name + 2) == 'i')
+        return 0x1000000;
+    /* Other names, "<Multi_key>", "<dead_*>", "<Cyrillic_*>" are described in
+     * the Compose file.
+     */
+    return 0;
+}
+
+
 static int
 ibus_compose_data_compare (gpointer a,
                            gpointer b,
@@ -619,6 +648,8 @@ ibus_compose_data_compare (gpointer a,
         gunichar code_a = compose_data_a->sequence[i];
         gunichar code_b = compose_data_b->sequence[i];
 
+        code_a += ibus_compose_key_flag (code_a);
+        code_b += ibus_compose_key_flag (code_b);
         if (code_a != code_b)
             return code_a - code_b;
         if (code_a == 0 && code_b == 0)
@@ -1542,13 +1573,16 @@ static int
 compare_seq (const void *key, const void *value)
 {
     int i = 0;
-    const guint16 *keysyms = key;
+    const guint *keysyms = key;
     const guint16 *seq = value;
 
     while (keysyms[i]) {
-        if (keysyms[i] < seq[i])
+        guint typed_key = keysyms[i];
+        guint saved_key = (guint)seq[i];
+        guint flag = ibus_compose_key_flag (saved_key);
+        if (typed_key < (saved_key + flag))
             return -1;
-        else if (keysyms[i] > seq[i])
+        else if (typed_key > (saved_key + flag))
             return 1;
 
         i++;
@@ -1558,9 +1592,40 @@ compare_seq (const void *key, const void *value)
 }
 
 
+/**
+ * ibus_compose_table_check:
+ * @table: An #IBusComposeTableEx.
+ * @compose_buffer: (array length=n_compose):
+                    A candidate typed key sequence to generate compose chars.
+ * @n_compose: The length of compose_buffer.
+ * @compose_finish: If typed key sequence is finished for the compose chars.
+ * @compose_match: If typed key sequence is matched partically.
+ * @output: Matched compse chars.
+ * @is_32bit: The type of #IBusComposeTableEx.
+ *
+ * If the current @comopse_buffer matches a compose sequence partically in
+ * #IBusComposeTableEx, return %TRUE otherwise %FALSE.
+ * If the matched compose sequence can be converted to a Unicode string,
+ * %TRUE is set to @compose_match and the converted string is assgined to
+ * @output to update the preedit text.
+ * If the current @comopse_buffer matches a compose sequence fully in
+ * #IBusComposeTableEx, %TRUE is set to @compose_finish and the generated
+ * compose character is assigned to @output.
+ *
+ * #IBusComposeTableEx includes 16bit key sequences and the corresponded
+ * compose characters. #IBusComposeTableEx has two types, one is
+ * 16bit and one compose char and another is 32bit or more than two chars.
+ * If #IBusComposeTableEx is 16bit and one compose char, @is_32bit is %TRUE.
+ *
+ * @compose_buffer is the typed key sequence but not limitted to 16bit.
+ * E.g.
+ * Shift + t is 0x100fef9 keysym with Arabic keyboard.
+ * Shift + NumLock is 0xfef9 keysym with keypad:pointerkeys XKB option.
+ * So compare_seq() compares the 32bit key sequence and 16bit one.
+ */
 gboolean
 ibus_compose_table_check (const IBusComposeTableEx *table,
-                          guint16                  *compose_buffer,
+                          guint                    *compose_buffer,
                           int                       n_compose,
                           gboolean                 *compose_finish,
                           gboolean                 *compose_match,
@@ -1749,7 +1814,7 @@ check_normalize_nfc (gunichar* combination_buffer, int n_compose)
 
 
 gboolean
-ibus_check_algorithmically (const guint16 *compose_buffer,
+ibus_check_algorithmically (const guint   *compose_buffer,
                             int            n_compose,
                             gunichar      *output_char)
 
@@ -1818,7 +1883,7 @@ ibus_check_algorithmically (const guint16 *compose_buffer,
 
 
 gunichar
-ibus_keysym_to_unicode (guint16   keysym,
+ibus_keysym_to_unicode (guint     keysym,
                         gboolean  combining,
                         gboolean *need_space) {
 #define CASE(keysym_suffix, unicode, sp)                                      \

src/ibusenginesimple.c

@@ -45,7 +45,7 @@
         COMPOSE_BUFFER_SIZE < IBUS_MAX_COMPOSE_LEN) {                   \
         COMPOSE_BUFFER_SIZE = ((index) + 10) < IBUS_MAX_COMPOSE_LEN     \
                               ? ((index) + 10) : IBUS_MAX_COMPOSE_LEN;  \
-        (buffer) = g_renew (guint16, (buffer), COMPOSE_BUFFER_SIZE + 1);\
+        (buffer) = g_renew (guint, (buffer), COMPOSE_BUFFER_SIZE + 1);  \
     }                                                                   \
     if ((index) < COMPOSE_BUFFER_SIZE) {                                \
         (buffer)[(index)] = (value);                                    \
@@ -70,7 +70,7 @@ typedef struct {
 } IBusEngineDict;
 
 struct _IBusEngineSimplePrivate {
-    guint16            *compose_buffer;
+    guint              *compose_buffer;
     GString            *tentative_match;
     int                 tentative_match_len;
     char               *tentative_emoji;
@@ -147,7 +147,7 @@ ibus_engine_simple_init (IBusEngineSimple *simple)
     IBusComposeTableEx *en_compose_table;
 
     simple->priv = IBUS_ENGINE_SIMPLE_GET_PRIVATE (simple);
-    simple->priv->compose_buffer = g_new0(guint16, COMPOSE_BUFFER_SIZE + 1);
+    simple->priv->compose_buffer = g_new0 (guint, COMPOSE_BUFFER_SIZE + 1);
     simple->priv->hex_mode_enabled =
         g_getenv("IBUS_ENABLE_CTRL_SHIFT_U") != NULL ||
         g_getenv("IBUS_ENABLE_CONTROL_SHIFT_U") != NULL;
@@ -326,7 +326,7 @@ ibus_engine_simple_update_preedit_text (IBusEngineSimple *simple)
             g_string_append (s, priv->tentative_match->str);
         } else {
             for (i = 0; priv->compose_buffer[i]; ++i) {
-                guint16 keysym = priv->compose_buffer[i];
+                guint keysym = priv->compose_buffer[i];
                 gboolean show_keysym = TRUE;
                 gboolean need_space = FALSE;
                 gunichar ch;
@@ -588,15 +588,15 @@ no_sequence_matches (IBusEngineSimple *simple,
      * match pending.
      */
     if (priv->tentative_match_len > 0) {
-        guint16 *compose_buffer;
+        guint *compose_buffer;
         int len = priv->tentative_match_len;
         int i;
         char *str;
 
-        compose_buffer = alloca (sizeof (guint16) * COMPOSE_BUFFER_SIZE);
+        compose_buffer = alloca (sizeof (guint) * COMPOSE_BUFFER_SIZE);
         memcpy (compose_buffer,
                 priv->compose_buffer,
-                sizeof (guint16) * COMPOSE_BUFFER_SIZE);
+                sizeof (guint) * COMPOSE_BUFFER_SIZE);
 
         str = g_strdup (priv->tentative_match->str);
         ibus_engine_simple_commit_str (simple, str);
@@ -649,9 +649,10 @@ no_sequence_matches (IBusEngineSimple *simple,
         ch = ibus_keyval_to_unicode (keyval);
         /* IBUS_CHANGE: RH#769133
          * Since we use ibus xkb engines as the disable state,
-         * do not commit the characters locally without in_hex_sequence. */
-        if (ch != 0 && !g_unichar_iscntrl (ch) &&
-            priv->in_hex_sequence) {
+         * Super-space and space key can launch IBus Emojier.
+         */
+        if (ch != 0 && !g_unichar_iscntrl (ch) && ch > 0x7F) {
+            ibus_engine_simple_commit_char (simple, ch);
             return TRUE;
         } else {
             return FALSE;

src/ibusenginesimpleprivate.h

@@ -42,7 +42,7 @@ struct _IBusComposeTablePrivate
 };
 
 
-gboolean ibus_check_algorithmically (const guint16              *compose_buffer,
+gboolean ibus_check_algorithmically (const guint                *compose_buffer,
                                      int                         n_compose,
                                      gunichar                   *output);
 GVariant *
@@ -56,13 +56,13 @@ IBusComposeTableEx *
                                      gsize                       length,
                                      gboolean                   reverse_endian);
 gboolean ibus_compose_table_check   (const IBusComposeTableEx   *table,
-                                     guint16                    *compose_buffer,
+                                     guint                      *compose_buffer,
                                      int                         n_compose,
                                      gboolean                   *compose_finish,
                                      gboolean                   *compose_match,
                                      GString                    *output,
                                      gboolean                    is_32bit);
-gunichar ibus_keysym_to_unicode     (guint16                     keysym,
+gunichar ibus_keysym_to_unicode     (guint                       keysym,
                                      gboolean                    combining,
                                      gboolean                   *need_space);
 

src/tests/Makefile.am

@@ -3,7 +3,7 @@
 # ibus - The Input Bus
 #
 # Copyright (c) 2007-2015 Peng Huang <shawn.p.huang@gmail.com>
-# Copyright (c) 2015-2022 Takao Fujiwara <takao.fujiwara1@gmail.com>
+# Copyright (c) 2015-2023 Takao Fujiwara <takao.fujiwara1@gmail.com>
 # Copyright (c) 2007-2018 Red Hat, Inc.
 #
 # This library is free software; you can redistribute it and/or
@@ -98,6 +98,7 @@ test_execs = $(TESTS:ibus-compose=ibus-compose-locales)
 test_metas = $(addsuffix .test, $(test_execs))
 test_sources_DATA = \
     $(test_metas) \
+    ibus-compose.basic \
     ibus-compose.emoji \
     ibus-compose.env \
     $(NULL)
@@ -151,6 +152,7 @@ EXTRA_DIST = \
     $(test_metas_in) \
     $(TESTS_SCRIPT) \
     runtest \
+    ibus-compose.basic \
     ibus-compose.emoji \
     ibus-compose.env \
     ibus-compose-locales.in \

src/tests/ibus-compose.basic

@@ -0,0 +1,24 @@
+# Symbol tests of Multi_key, quotedbl Cyrillic_zhe
+# This case swaps U04DC and U04DD in en-US
+<Multi_key> <quotedbl> <Cyrillic_zhe>   : "Ӝ"   U04DC
+<Multi_key> <quotedbl> <Cyrillic_ZHE>   : "ӝ"   U04DD
+#
+# Unicode tests of Uxxxx
+# en-US is "ǡ" U01E1 but this case is "ǟ" U01DE
+<dead_macron> <U0227>                           : "ǟ"   U01DE
+<Multi_key> <macron> <U0227>                    : "ǟ"   U01DE
+<Multi_key> <underscore> <U0227>                : "ǟ"   U01DE
+# Khmer digraphs
+# This case swaps U17fe and U17ff in en-US
+<U17fe> : "ាំ"
+<U17ff> : "ោះ"
+# Arabic Lam-Alef ligatures
+# This case swaps UFEF9 and UFEFB in en-US
+<UFEF9>	: "لا" # ARABIC LIGATURE LAM WITH ALEF
+<UFEFB>	: "لإ" # ARABIC LIGATURE LAM WITH ALEF WITH HAMZA BELOW
+#
+# Symbol tests of c_h, C_h
+# Breton N-graphs
+# This case swaps c_h and C_h in en-US
+<c_h>   : "C’h"
+<C_h>   : "c’h"

src/tests/ibus-compose.c

@@ -13,6 +13,8 @@ IBusComposeTableEx *m_compose_table;
 IBusEngine *m_engine;
 gchar *m_srcdir;
 
+guint ibus_compose_key_flag (guint key);
+
 static gboolean window_focus_in_event_cb (GtkWidget     *entry,
                                           GdkEventFocus *event,
                                           gpointer       data);
@@ -163,6 +165,7 @@ set_engine_cb (GObject *object, GAsyncResult *res, gpointer data)
 
             if (keyval == 0)
                 break;
+            keyval += ibus_compose_key_flag (keyval);
             g_signal_emit_by_name (m_engine, "process-key-event",
                                    keyval, keycode, modifiers, &retval);
             modifiers |= IBUS_RELEASE_MASK;
@@ -183,6 +186,7 @@ set_engine_cb (GObject *object, GAsyncResult *res, gpointer data)
 
                 if (keyval == 0)
                     break;
+                keyval += ibus_compose_key_flag (keyval);
                 g_signal_emit_by_name (m_engine, "process-key-event",
                                        keyval, keycode, modifiers, &retval);
                 modifiers |= IBUS_RELEASE_MASK;
@@ -245,6 +249,8 @@ window_inserted_text_cb (GtkEntryBuffer *buffer,
 #endif
     i = stride + (m_compose_table->max_seq_len + 2) - 2;
     seq = (i + 2) / (m_compose_table->max_seq_len + 2);
+    if (!enable_32bit && !m_compose_table->n_seqs && priv)
+        enable_32bit = TRUE;
     if (!enable_32bit) {
         if (m_compose_table->data[i] == code) {
             test = GREEN "PASS" NC;
@@ -294,6 +300,7 @@ window_inserted_text_cb (GtkEntryBuffer *buffer,
         if (priv) {
             enable_32bit = TRUE;
             stride = 0;
+            seq = 0;
         } else {
             gtk_main_quit ();
             return;
@@ -353,7 +360,7 @@ test_compose (void)
 int
 main (int argc, char *argv[])
 {
-    const gchar *test_name;
+    gchar *test_name;
     gchar *test_path;
 
     ibus_init ();
@@ -369,13 +376,18 @@ main (int argc, char *argv[])
             ? g_strdup (argv[1]) : g_strdup (".");
     m_compose_file = g_strdup (g_getenv ("COMPOSE_FILE"));
 #if GLIB_CHECK_VERSION (2, 58, 0)
-    test_name = g_get_language_names_with_category ("LC_CTYPE")[0];
+    test_name = g_strdup (g_get_language_names_with_category ("LC_CTYPE")[0]);
 #else
-    test_name = g_getenv ("LANG");
+    test_name = g_strdup (g_getenv ("LANG"));
 #endif
+    if (!test_name || !g_ascii_strncasecmp (test_name, "en_US", 5)) {
+        g_free (test_name);
+        test_name = g_path_get_basename (m_compose_file);
+    }
     test_path = g_build_filename ("/ibus-compose", test_name, NULL);
     g_test_add_func (test_path, test_compose);
     g_free (test_path);
+    g_free (test_name);
 
     return g_test_run ();
 }

src/tests/ibus-compose.env

@@ -1,4 +1,5 @@
 LANG=el_GR.UTF-8
 LANG=fi_FI.UTF-8
 LANG=pt_BR.UTF-8
+LANG=en_US.UTF-8 COMPOSE_FILE=ibus-compose.basic
 LANG=en_US.UTF-8 COMPOSE_FILE=ibus-compose.emoji