feat(tui): query terminal for CSI u support (neovim#18264)

On startup query the terminal for CSI u support and enable it using
the escape sequence from kitty's progressive enhancement protocol [1].

[1]: https://sw.kovidgoyal.net/kitty/keyboard-protocol/

(cherry picked from commit 797a252)

Co-authored-by: Gregory Anders <greg@gpanders.com>

runtime/doc/term.txt

@@ -117,6 +117,43 @@ go to the window below: >
   tmux send-keys 'Escape' [ 2 7 u 'C-W' j
 Where `'Escape' [ 2 7 u` is an unambiguous `CSI u` sequence for the <Esc> key.
 
+						*tui-modifyOtherKeys* *tui-csiu*
+Historically, terminal emulators could not distinguish between certain control
+key modifiers and other keys. For example, <C-I> and <Tab> are represented the
+same way, as are <Esc> and <C-[>, <CR> and <C-M>, and <NL> and <C-J>. This
+meant that Nvim also could not map these keys separately.
+
+Modern terminal emulators are able to distinguish between these pairs of keys
+by encoding control modifiers differently. There are two common but distinct
+ways of doing this, known as "modifyOtherKeys" and "CSI u". Nvim supports both
+encoding methods and at startup will tell the terminal emulator that it
+understands these key encodings. If your terminal emulator supports it then
+this will allow you to map the key pairs listed above separately.
+
+At startup Nvim will query your terminal to see if it supports the CSI u
+encoding by writing the sequence >
+
+	CSI ? u CSI c
+
+If your terminal emulator responds with >
+
+	CSI ? <flags> u
+
+this means your terminal supports the CSI u encoding and Nvim will tell your
+terminal to enable it by writing the sequence >
+
+	CSI > 1 u
+
+If your terminal does not support CSI u then Nvim will instead enable the
+"modifyOtherKeys" encoding by writing the sequence >
+
+	CSI > 4 ; 2 m
+
+When Nvim exits cleanly it will send the corresponding sequence to disable the
+special key encoding. If Nvim does not exit cleanly then your terminal
+emulator could be in a bad state. If this happens, simply run "reset".
+
+
 							*tui-colors*
 Nvim uses 256 colours by default, ignoring |terminfo| for most terminal types,
 including "linux" (whose virtual terminals have had 256-colour support since

src/nvim/tui/input.c

@@ -13,6 +13,7 @@
 #include "nvim/option.h"
 #include "nvim/os/input.h"
 #include "nvim/os/os.h"
+#include "nvim/tui/tui.h"
 #include "nvim/tui/input.h"
 #include "nvim/vim.h"
 #ifdef WIN32
@@ -41,6 +42,7 @@ void tinput_init(TermInput *input, Loop *loop)
   input->paste = 0;
   input->in_fd = STDIN_FILENO;
   input->waiting_for_bg_response = 0;
+  input->extkeys_type = kExtkeysNone;
   // The main thread is waiting for the UI thread to call CONTINUE, so it can
   // safely access global variables.
   input->ttimeout = (bool)p_ttimeout;
@@ -344,6 +346,39 @@ static void tk_getkeys(TermInput *input, bool force)
       forward_modified_utf8(input, &key);
     } else if (key.type == TERMKEY_TYPE_MOUSE) {
       forward_mouse_event(input, &key);
+    } else if (key.type == TERMKEY_TYPE_UNKNOWN_CSI) {
+      // There is no specified limit on the number of parameters a CSI sequence can contain, so just
+      // allocate enough space for a large upper bound
+      long args[16];
+      size_t nargs = 16;
+      unsigned long cmd;
+      if (termkey_interpret_csi(input->tk, &key, args, &nargs, &cmd) == TERMKEY_RES_KEY) {
+        uint8_t intermediate = (cmd >> 16) & 0xFF;
+        uint8_t initial = (cmd >> 8) & 0xFF;
+        uint8_t command = cmd & 0xFF;
+
+        // Currently unused
+        (void)intermediate;
+
+        if (input->waiting_for_csiu_response > 0) {
+          if (initial == '?' && command == 'u') {
+            // The first (and only) argument contains the current progressive
+            // enhancement flags. Only enable CSI u mode if the first bit
+            // (disambiguate escape codes) is not already set
+            if (nargs > 0 && (args[0] & 0x1) == 0) {
+              input->extkeys_type = kExtkeysCSIu;
+            } else {
+              input->extkeys_type = kExtkeysNone;
+            }
+          } else if (initial == '?' && command == 'c') {
+            // Received Primary Device Attributes response
+            input->waiting_for_csiu_response = 0;
+            tui_enable_extkeys(input->tui_data);
+          } else {
+            input->waiting_for_csiu_response--;
+          }
+        }
+      }
     }
   }
 

src/nvim/tui/input.h

@@ -6,6 +6,13 @@
 
 #include "nvim/event/stream.h"
 #include "nvim/event/time.h"
+#include "nvim/tui/tui.h"
+
+typedef enum {
+  kExtkeysNone,
+  kExtkeysCSIu,
+  kExtkeysXterm,
+} ExtkeysType;
 
 typedef struct term_input {
   int in_fd;
@@ -14,6 +21,8 @@ typedef struct term_input {
   bool waiting;
   bool ttimeout;
   int8_t waiting_for_bg_response;
+  int8_t waiting_for_csiu_response;
+  ExtkeysType extkeys_type;
   long ttimeoutlen;
   TermKey *tk;
 #if TERMKEY_VERSION_MAJOR > 0 || TERMKEY_VERSION_MINOR > 18
@@ -25,6 +34,7 @@ typedef struct term_input {
   RBuffer *key_buffer;
   uv_mutex_t key_buffer_mutex;
   uv_cond_t key_buffer_cond;
+  TUIData *tui_data;
 } TermInput;
 
 #ifdef INCLUDE_GENERATED_DECLARATIONS

src/nvim/tui/tui.c

@@ -71,7 +71,7 @@ typedef struct {
   int top, bot, left, right;
 } Rect;
 
-typedef struct {
+struct TUIData {
   UIBridgeData *bridge;
   Loop *loop;
   unibi_var_t params[9];
@@ -132,9 +132,10 @@ typedef struct {
     int set_underline_style;
     int set_underline_color;
     int enable_extended_keys, disable_extended_keys;
+    int get_extkeys;
   } unibi_ext;
   char *space_buf;
-} TUIData;
+};
 
 static bool volatile got_winch = false;
 static bool did_user_set_dimensions = false;
@@ -179,6 +180,32 @@ UI *tui_start(void)
   return ui_bridge_attach(ui, tui_main, tui_scheduler);
 }
 
+void tui_enable_extkeys(TUIData *data)
+{
+  TermInput input = data->input;
+  unibi_term *ut = data->ut;
+  UI *ui = data->bridge->ui;
+
+  switch (input.extkeys_type) {
+  case kExtkeysCSIu:
+    data->unibi_ext.enable_extended_keys = (int)unibi_add_ext_str(ut, "ext.enable_extended_keys",
+                                                                  "\x1b[>1u");
+    data->unibi_ext.disable_extended_keys = (int)unibi_add_ext_str(ut, "ext.disable_extended_keys",
+                                                                   "\x1b[<1u");
+    break;
+  case kExtkeysXterm:
+    data->unibi_ext.enable_extended_keys = (int)unibi_add_ext_str(ut, "ext.enable_extended_keys",
+                                                                  "\x1b[>4;2m");
+    data->unibi_ext.disable_extended_keys = (int)unibi_add_ext_str(ut, "ext.disable_extended_keys",
+                                                                   "\x1b[>4;0m");
+    break;
+  default:
+    break;
+  }
+
+  unibi_out_ext(ui, data->unibi_ext.enable_extended_keys);
+}
+
 static size_t unibi_pre_fmt_str(TUIData *data, unsigned int unibi_index, char *buf, size_t len)
 {
   const char *str = unibi_get_str(data->ut, unibi_index);
@@ -228,8 +255,10 @@ static void terminfo_start(UI *ui)
   data->unibi_ext.set_underline_color = -1;
   data->unibi_ext.enable_extended_keys = -1;
   data->unibi_ext.disable_extended_keys = -1;
+  data->unibi_ext.get_extkeys = -1;
   data->out_fd = STDOUT_FILENO;
   data->out_isatty = os_isatty(data->out_fd);
+  data->input.tui_data = data;
 
   const char *term = os_getenv("TERM");
 #ifdef WIN32
@@ -311,8 +340,9 @@ static void terminfo_start(UI *ui)
   // Enable bracketed paste
   unibi_out_ext(ui, data->unibi_ext.enable_bracketed_paste);
 
-  // Enable extended keys (also known as 'modifyOtherKeys' or CSI u)
-  unibi_out_ext(ui, data->unibi_ext.enable_extended_keys);
+  // Query the terminal to see if it supports CSI u
+  data->input.waiting_for_csiu_response = 5;
+  unibi_out_ext(ui, data->unibi_ext.get_extkeys);
 
   int ret;
   uv_loop_init(&data->write_loop);
@@ -1810,6 +1840,12 @@ static void patch_terminfo_bugs(TUIData *data, const char *term, const char *col
   data->unibi_ext.get_bg = (int)unibi_add_ext_str(ut, "ext.get_bg",
                                                   "\x1b]11;?\x07");
 
+  // Query the terminal to see if it supports CSI u key encoding by writing CSI
+  // ? u followed by a request for the primary device attributes (CSI c)
+  // See https://sw.kovidgoyal.net/kitty/keyboard-protocol/#detection-of-support-for-this-protocol
+  data->unibi_ext.get_extkeys = (int)unibi_add_ext_str(ut, "ext.get_extkeys",
+                                                       "\x1b[?u\x1b[c");
+
   // Terminals with 256-colour SGR support despite what terminfo says.
   if (unibi_get_num(ut, unibi_max_colors) < 256) {
     // See http://fedoraproject.org/wiki/Features/256_Color_Terminals
@@ -2074,15 +2110,9 @@ static void augment_terminfo(TUIData *data, const char *term, long vte_version,
                                                                  "\x1b[58:2::%p1%d:%p2%d:%p3%dm");
   }
 
-  data->unibi_ext.enable_extended_keys = unibi_find_ext_str(ut, "Eneks");
-  data->unibi_ext.disable_extended_keys = unibi_find_ext_str(ut, "Dseks");
-  if (data->unibi_ext.enable_extended_keys == -1) {
-    if (!kitty && (vte_version == 0 || vte_version >= 5400)) {
-      data->unibi_ext.enable_extended_keys = (int)unibi_add_ext_str(ut, "ext.enable_extended_keys",
-                                                                    "\x1b[>4;2m");
-      data->unibi_ext.disable_extended_keys = (int)unibi_add_ext_str(ut, "ext.disable_extended_keys",
-                                                                     "\x1b[>4m");
-    }
+  if (!kitty && (vte_version == 0 || vte_version >= 5400)) {
+    // Fallback to Xterm's modifyOtherKeys if terminal does not support CSI u
+    data->input.extkeys_type = kExtkeysXterm;
   }
 }
 

src/nvim/tui/tui.h

@@ -4,6 +4,8 @@
 #include "nvim/cursor_shape.h"
 #include "nvim/ui.h"
 
+typedef struct TUIData TUIData;
+
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "tui/tui.h.generated.h"
 #endif

test/functional/core/main_spec.lua

@@ -52,11 +52,15 @@ describe('Command-line option', function()
       if helpers.pending_win32(pending) then return end
       local screen = Screen.new(40, 8)
       screen:attach()
-      funcs.termopen({
+      local args = {
         nvim_prog_abs(), '-u', 'NONE', '-i', 'NONE',
-         '--cmd', 'set noswapfile shortmess+=IFW fileformats=unix',
-         '-s', '-'
-      })
+        '--cmd', 'set noswapfile shortmess+=IFW fileformats=unix',
+        '-s', '-'
+      }
+
+      -- Need to explicitly pipe to stdin so that the embedded Nvim instance doesn't try to read
+      -- data from the terminal #18181
+      funcs.termopen(string.format([[echo "" | %s]], table.concat(args, " ")))
       screen:expect([[
         ^                                        |
         {1:~                                       }|