patch 9.0.0930: cannot debug the Kitty keyboard protocol with TermDebug

Problem:    Cannot debug the Kitty keyboard protocol with TermDebug.
Solution:   Add Kitty keyboard protocol support to the libvterm fork.
            Recognize the escape sequences that the protocol generates.  Add
            the 'keyprotocol' option to allow the user to specify for which
            terminal what protocol is to be used, instead of hard-coding this.
            Add recognizing the kitty keyboard protocol status.

Author: brammool

runtime/doc/map.txt

@@ -10,18 +10,19 @@ This subject is introduced in sections |05.4|, |24.7| and |40.1| of the user
 manual.
 
 1. Key mapping			|key-mapping|
-   1.1 MAP COMMANDS			|:map-commands|
-   1.2 Special arguments		|:map-arguments|
-   1.3 Mapping and modes		|:map-modes|
-   1.4 Listing mappings			|map-listing|
-   1.5 Mapping special keys		|:map-special-keys|
-   1.6 Special characters		|:map-special-chars|
-   1.7 What keys to map			|map-which-keys|
-   1.8 Examples				|map-examples|
-   1.9 Using mappings			|map-typing|
-   1.10 Mapping alt-keys		|:map-alt-keys|
-   1.11 Mapping in modifyOtherKeys mode	|modifyOtherKeys|
-   1.12 Mapping an operator		|:map-operator|
+   1.1 MAP COMMANDS				|:map-commands|
+   1.2 Special arguments			|:map-arguments|
+   1.3 Mapping and modes			|:map-modes|
+   1.4 Listing mappings				|map-listing|
+   1.5 Mapping special keys			|:map-special-keys|
+   1.6 Special characters			|:map-special-chars|
+   1.7 What keys to map				|map-which-keys|
+   1.8 Examples					|map-examples|
+   1.9 Using mappings				|map-typing|
+   1.10 Mapping alt-keys			|:map-alt-keys|
+   1.11 Mapping in modifyOtherKeys mode		|modifyOtherKeys|
+   1.12 Mapping with Kitty keyboard protocol	|kitty-keyboard-protocol|
+   1.13 Mapping an operator			|:map-operator|
 2. Abbreviations		|abbreviations|
 3. Local mappings and functions	|script-local|
 4. User-defined commands	|user-commands|
@@ -1009,7 +1010,34 @@ When the 'esckeys' option is off, then modifyOtherKeys will be disabled in
 Insert mode to avoid every key with a modifier causing Insert mode to end.
 
 
-1.12 MAPPING AN OPERATOR				*:map-operator*
+1.12 MAPPING WITH KITTY KEYBOARD PROTOCOL	 *kitty-keyboard-protocol*
+
+If the value of 'term' contains "kitty" then Vim will send out an escape
+sequence to enable the Kitty keyboard protocol.  This can be changed with the
+'keyprotocol' option.
+
+Like modifyOtherKeys, this will make it possible to distinguish between more
+keys with modifiers.  Also, this protocol sends an escape sequence for the Esc
+key, so that Vim does not need to use a timeout to know whether receiving an
+Esc character means the Esc key was pressed or it's the start of an escape
+sequence.
+
+Vim automatically detects if the Kitty keyboard protocol was enabled when it
+spots the response to the status request (this should be part of the |t_TI|
+termcap entry).  To see if Vim detected such an escape sequence use: >
+	:verbose map
+The first line will then show "Kitty keyboard protocol: {value}" (possibly
+translated).  The meaning of {value}:
+	Unknown		no status received yet
+	Off		protocol is not used
+	On		protocol is used
+	Disabled	protocol was used but expected to have been disabled
+			by 't_TE'
+	Cleared		protocol expected to have beeen disabled by 't_TE',
+			previous state is unknown 
+
+
+1.13 MAPPING AN OPERATOR				*:map-operator*
 
 An operator is used before a {motion} command.  To define your own operator
 you must create a mapping that first sets the 'operatorfunc' option and then

runtime/doc/options.txt

@@ -4888,6 +4888,50 @@ A jump table for the options with a short description can be found at |Q_op|.
 	<PageUp> and <PageDown>.
 	The 'keymodel' option is set by the |:behave| command.
 
+						*'keyprotocol'* *'kpc'*
+'keyprotocol' 'kpc'	string	(default: see below)
+			global
+	Specifies what keyboard protocol to use depending on the value of
+	'term'.  The supported keyboard protocols names are:
+		none	whatever the terminal uses
+		mok2	modifyOtherKeys level 2, as supported by xterm
+		kitty	Kitty keyboard protocol, as supported by Kitty
+
+	The option value is a list of command separated items.  Each item has
+	a pattern that is matched against the 'term' option, a colon and the
+	protocol name to be used.  To illustrate this, the default value would
+	be set with: >
+	   set keyprotocol=kitty:kitty,foot:kitty,wezterm:kitty,xterm:mok2
+
+<	This means that when 'term' contains "kitty, "foot" or "wezterm"
+	somewhere then the "kitty" protocol is used.  When 'term' contains
+	"xterm" somewhere, then the "mok2" protocol is used. 
+
+	The first match is used, thus if you want to have "kitty" use the
+	kitty protocol, but "badkitty" not, then you should match "badkitty"
+	first and use the "none" value: >
+	   set keyprotocol=badkitty:none,kitty:kitty
+<
+	The option is used after 'term' has been changed.  First the termcap
+	entries are set, possibly using the builtin list, see |builtin-terms|.
+	Then this option is inspected and if there is a match and a protocol
+	is specified the following happens:
+		none	Nothing, the regular t_TE and t_TI values remain
+
+		mok2	The t_TE value is changed to:
+			    CSI >4;m	disables modifyOtherKeys
+			The t_TI value is changed to:
+			    CSI >4;2m	enables modifyOtherKeys
+
+		kitty	The t_TE value is changed to:
+			    CSI >4;m	disables modifyOtherKeys
+			    CSI <u	disables the kitty keyboard protocol
+			The t_TI value is changed to:
+			    CSI >1u	enables the kitty keyboard protocol
+			    CSI ?u	request kitty keyboard protocol state
+			    CSI >c	request the termresponse
+
+
 					*'keywordprg'* *'kp'*
 'keywordprg' 'kp'	string	(default "man" or "man -s",  DOS: ":help",
 								  VMS: "help")

runtime/doc/quickref.txt

@@ -775,6 +775,7 @@ Short explanation of each option:		*option-list*
 'key'			    encryption key
 'keymap'	  'kmp'     name of a keyboard mapping
 'keymodel'	  'km'	    enable starting/stopping selection with keys
+'keyprotocol'	  'kpc'	    what keyboard protocol to use for what terminal
 'keywordprg'	  'kp'	    program to use for the "K" command
 'langmap'	  'lmap'    alphabetic characters for other language mode
 'langmenu'	  'lm'	    language to be used for the menus

src/edit.c

@@ -330,7 +330,7 @@ edit(
 
 	// Disable modifyOtherKeys, keys with modifiers would cause exiting
 	// Insert mode.
-	out_str(T_CTE);
+	out_str_t_TE();
     }
 
     /*

src/globals.h

@@ -1377,6 +1377,24 @@ EXTERN int pending_end_reg_executing INIT(= 0);
 // no longer be used.
 EXTERN int seenModifyOtherKeys INIT(= FALSE);
 
+// The state for the Kitty keyboard protocol.
+typedef enum {
+    // Initially we have no clue if the protocol is on or off.
+    KKPS_INITIAL,
+    // Used when receiving the state and the flags are zero.
+    KKPS_OFF,
+    // Used when receiving the state and the flags are non-zero.
+    KKPS_ENABLED,
+    // Used after outputting t_KE when the state was KKPS_ENABLED.  We do not
+    // really know if t_KE actually disabled the protocol, the following t_KI
+    // is expected to request the state, but the response may come only later.
+    KKPS_DISABLED,
+    // Used after outputting t_KE when the state was not KKPS_ENABLED.
+    KKPS_AFTER_T_KE,
+} kkpstate_T;
+
+EXTERN kkpstate_T kitty_protocol_state INIT(= KKPS_INITIAL);
+
 EXTERN int no_mapping INIT(= FALSE);	// currently no mapping allowed
 EXTERN int no_zero_mapping INIT(= 0);	// mapping zero not allowed
 EXTERN int allow_keys INIT(= FALSE);	// allow key codes when no_mapping

src/libvterm/include/vterm.h

@@ -368,6 +368,7 @@ size_t vterm_output_get_buffer_remaining(const VTerm *vt);
 size_t vterm_output_read(VTerm *vt, char *buffer, size_t len);
 
 int vterm_is_modify_other_keys(VTerm *vt);
+int vterm_is_kitty_keyboard(VTerm *vt);
 void vterm_keyboard_unichar(VTerm *vt, uint32_t c, VTermModifier mod);
 void vterm_keyboard_key(VTerm *vt, VTermKey key, VTermModifier mod);
 

src/libvterm/src/keyboard.c

@@ -10,6 +10,12 @@ int vterm_is_modify_other_keys(VTerm *vt)
   return vt->state->mode.modify_other_keys;
 }
 
+// VIM: added
+int vterm_is_kitty_keyboard(VTerm *vt)
+{
+  return vt->state->mode.kitty_keyboard;
+}
+
 
 void vterm_keyboard_unichar(VTerm *vt, uint32_t c, VTermModifier mod)
 {
@@ -19,6 +25,12 @@ void vterm_keyboard_unichar(VTerm *vt, uint32_t c, VTermModifier mod)
     return;
   }
 
+  // VIM: added kitty keyboard protocol support
+  if (vterm_is_kitty_keyboard(vt) && mod != 0) {
+    vterm_push_output_sprintf_ctrl(vt, C1_CSI, "%d;%du", c, mod+1);
+    return;
+  }
+
   /* The shift modifier is never important for Unicode characters
    * apart from Space
    */
@@ -166,8 +178,10 @@ void vterm_keyboard_key(VTerm *vt, VTermKey key, VTermModifier mod)
     break;
 
   case KEYCODE_TAB:
+    if (vterm_is_kitty_keyboard(vt) && mod != 0)
+      vterm_push_output_sprintf_ctrl(vt, C1_CSI, "9;%du", mod+1);
     /* Shift-Tab is CSI Z but plain Tab is 0x09 */
-    if(mod == VTERM_MOD_SHIFT)
+    else if(mod == VTERM_MOD_SHIFT)
       vterm_push_output_sprintf_ctrl(vt, C1_CSI, "Z");
     else if(mod & VTERM_MOD_SHIFT)
       vterm_push_output_sprintf_ctrl(vt, C1_CSI, "1;%dZ", mod+1);
@@ -186,7 +200,10 @@ void vterm_keyboard_key(VTerm *vt, VTermKey key, VTermModifier mod)
   case KEYCODE_LITERAL: case_LITERAL:
     if (vterm_is_modify_other_keys(vt) && mod != 0)
       vterm_push_output_sprintf_ctrl(vt, C1_CSI, "27;%d;%d~", mod+1, k.literal);
-    else if(mod & (VTERM_MOD_SHIFT|VTERM_MOD_CTRL))
+    else if (vterm_is_kitty_keyboard(vt) && mod == 0 && k.literal == '\x1b')
+      vterm_push_output_sprintf_ctrl(vt, C1_CSI, "%du", k.literal);
+    else if ((vterm_is_kitty_keyboard(vt) && mod != 0)
+	|| (mod & (VTERM_MOD_SHIFT|VTERM_MOD_CTRL)))
       vterm_push_output_sprintf_ctrl(vt, C1_CSI, "%d;%du", k.literal, mod+1);
     else
       vterm_push_output_sprintf(vt, mod & VTERM_MOD_ALT ? ESC_S "%c" : "%c", k.literal);

src/libvterm/src/state.c

@@ -1423,9 +1423,37 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
     }
     break;
 
-  case LEADER('>', 0x6d): // xterm resource modifyOtherKeys
+  case LEADER('>', 0x6d): // CSI > 4 ; Pv m   xterm resource modifyOtherKeys
     if (argcount == 2 && args[0] == 4)
+    {
+      // can't have both modify_other_keys and kitty_keyboard
+      state->mode.kitty_keyboard = 0;
+
       state->mode.modify_other_keys = args[1] == 2;
+    }
+    break;
+
+  case LEADER('>', 0x75): // CSI > 1 u  enable kitty keyboard protocol
+    if (argcount == 1 && args[0] == 1)
+    {
+      // can't have both modify_other_keys and kitty_keyboard
+      state->mode.modify_other_keys = 0;
+
+      state->mode.kitty_keyboard = 1;
+    }
+    break;
+
+  case LEADER('<', 0x75): // CSI < u  disable kitty keyboard protocol
+    if (argcount <= 1)
+      state->mode.kitty_keyboard = 0;
+    break;
+
+  case LEADER('?', 0x75): // CSI ? u  request kitty keyboard protocol state
+    if (argcount <= 1)
+      // TODO: this only uses the values zero and one.  The protocol specifies
+      // more values, the progressive enhancement flags.
+      vterm_push_output_sprintf_ctrl(state->vt, C1_CSI, "?%du",
+						   state->mode.kitty_keyboard);
     break;
 
   case 0x6e: // DSR - ECMA-48 8.3.35

src/libvterm/src/vterm_internal.h

@@ -130,6 +130,7 @@ struct VTermState
     unsigned int bracketpaste:1;
     unsigned int report_focus:1;
     unsigned int modify_other_keys:1;
+    unsigned int kitty_keyboard:1;
   } mode;
 
   VTermEncodingInstance encoding[4], encoding_utf8;

src/map.c

@@ -312,8 +312,27 @@ list_mappings(
     // Prevent mappings to be cleared while at the more prompt.
     ++map_locked;
 
-    if (p_verbose > 0 && keyround == 1 && seenModifyOtherKeys)
-	msg_puts(_("Seen modifyOtherKeys: true"));
+    if (p_verbose > 0 && keyround == 1)
+    {
+	if (seenModifyOtherKeys)
+	    msg_puts(_("Seen modifyOtherKeys: true"));
+	if (kitty_protocol_state != KKPS_INITIAL)
+	{
+	    char *name = _("Unknown");
+	    switch (kitty_protocol_state)
+	    {
+		case KKPS_INITIAL: break;
+		case KKPS_OFF: name = _("Off"); break;
+		case KKPS_ENABLED: name = _("On"); break;
+		case KKPS_DISABLED: name = _("Disabled"); break;
+		case KKPS_AFTER_T_KE: name = _("Cleared"); break;
+	    }
+
+	    char buf[200];
+	    vim_snprintf(buf, sizeof(buf), _("Kitty keyboard protocol: %s"), name);
+	    msg_puts(buf);
+	}
+    }
 
     // need to loop over all global hash lists
     for (int hash = 0; hash < 256 && !got_int; ++hash)