Skip to content
Permalink
Browse files
Keyman: Improved tokenization (#3224)
  • Loading branch information
RunDevelopment committed Dec 5, 2021
1 parent d04d166 commit baa95cab5f1f69aa4b965d7335b657b2a2931959
@@ -1,14 +1,44 @@
Prism.languages.keyman = {
'comment': /\bc\s.*/i,
'function': /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|".+?"|'.+?')\s*\]/i, // virtual key
'string': /("|').*?\1/,
'bold': [ // header statements, system stores and variable system stores
/&(?:baselayout|bitmap|capsalwaysoff|capsononly|copyright|ethnologuecode|hotkey|includecodes|keyboardversion|kmw_embedcss|kmw_embedjs|kmw_helpfile|kmw_helptext|kmw_rtl|language|layer|layoutfile|message|mnemoniclayout|name|oldcharposmatching|platform|shiftfreescaps|targets|version|visualkeyboard|windowslanguages)\b/i,
/\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i
],
'keyword': /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i, // rule keywords
'atrule': /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i, // structural keywords
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i, // U+####, x###, d### characters and numbers
'operator': /[+>\\,()]/,
'tag': /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i // prefixes
'comment': {
pattern: /\bc .*/i,
greedy: true
},
'string': {
pattern: /"[^"\r\n]*"|'[^'\r\n]*'/,
greedy: true
},
'virtual-key': {
pattern: /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|[A-E]\d\d?|"[^"\r\n]*"|'[^'\r\n]*')\s*\]/i,
greedy: true,
alias: 'function' // alias for styles
},

// https://help.keyman.com/developer/language/guide/headers
'header-keyword': {
pattern: /&\w+/,
alias: 'bold' // alias for styles
},
'header-statement': {
pattern: /\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i,
alias: 'bold' // alias for styles
},

'rule-keyword': {
pattern: /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i,
alias: 'keyword'
},
'structural-keyword': {
pattern: /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i,
alias: 'keyword'
},

'compile-target': {
pattern: /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i,
alias: 'property'
},

// U+####, x###, d### characters and numbers
'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i,
'operator': /[+>\\$]|\.\./,
'punctuation': /[()=,]/
};

Some generated files are not rendered by default. Learn more.

This file was deleted.

This file was deleted.

@@ -0,0 +1,27 @@
$keyman:
$kmfl:
$weaver:
$keymanweb:
$keymanonly:

c Note: return statement is only supported in .kmx in Keyman 13
$keymanonly: + 'a' > 'a' return

----------------------------------------------------

[
["compile-target", "$keyman:"],
["compile-target", "$kmfl:"],
["compile-target", "$weaver:"],
["compile-target", "$keymanweb:"],
["compile-target", "$keymanonly:"],

["comment", "c Note: return statement is only supported in .kmx in Keyman 13"],

["compile-target", "$keymanonly:"],
["operator", "+"],
["string", "'a'"],
["operator", ">"],
["string", "'a'"],
["rule-keyword", "return"]
]

This file was deleted.

@@ -0,0 +1,59 @@
&baselayout
&bitmap
&capsononly
&capsalwaysoff
&shiftfreescaps
&copyright
&ethnologuecode
&hotkey
&includecodes
&keyboardversion
&kmw_embedcss
&kmw_embedjs
&kmw_helpfile
&kmw_helptext
&kmw_rtl
&language
&layer
&layoutfile
&message
&mnemoniclayout
&name
&oldcharposmatching
&platform
&targets
&version
&visualkeyboard
&windowslanguages

----------------------------------------------------

[
["header-keyword", "&baselayout"],
["header-keyword", "&bitmap"],
["header-keyword", "&capsononly"],
["header-keyword", "&capsalwaysoff"],
["header-keyword", "&shiftfreescaps"],
["header-keyword", "&copyright"],
["header-keyword", "&ethnologuecode"],
["header-keyword", "&hotkey"],
["header-keyword", "&includecodes"],
["header-keyword", "&keyboardversion"],
["header-keyword", "&kmw_embedcss"],
["header-keyword", "&kmw_embedjs"],
["header-keyword", "&kmw_helpfile"],
["header-keyword", "&kmw_helptext"],
["header-keyword", "&kmw_rtl"],
["header-keyword", "&language"],
["header-keyword", "&layer"],
["header-keyword", "&layoutfile"],
["header-keyword", "&message"],
["header-keyword", "&mnemoniclayout"],
["header-keyword", "&name"],
["header-keyword", "&oldcharposmatching"],
["header-keyword", "&platform"],
["header-keyword", "&targets"],
["header-keyword", "&version"],
["header-keyword", "&visualkeyboard"],
["header-keyword", "&windowslanguages"]
]
@@ -0,0 +1,29 @@
bitmap
bitmaps
caps on only
caps always off
shift frees caps
copyright
hotkey
language
layout
message
name
version

----------------------------------------------------

[
["header-statement", "bitmap"],
["header-statement", "bitmaps"],
["header-statement", "caps on only"],
["header-statement", "caps always off"],
["header-statement", "shift frees caps"],
["header-statement", "copyright"],
["header-statement", "hotkey"],
["header-statement", "language"],
["header-statement", "layout"],
["header-statement", "message"],
["header-statement", "name"],
["header-statement", "version"]
]

This file was deleted.

@@ -1,15 +1,17 @@
+ >
\ ,
( )
+ > \

$vowel_a
'a' .. 'z'

----------------------------------------------------

[
["operator", "+"], ["operator", ">"],
["operator", "\\"], ["operator", ","],
["operator", "("], ["operator", ")"]
["operator", "+"], ["operator", ">"], ["operator", "\\"],

["operator", "$"], "vowel_a\r\n",
["string", "'a'"], ["operator", ".."], ["string", "'z'"]
]

----------------------------------------------------

Checks for all operators.
Checks for all operators.
@@ -0,0 +1,9 @@
( )
, =

----------------------------------------------------

[
["punctuation", "("], ["punctuation", ")"],
["punctuation", ","], ["punctuation", "="]
]

0 comments on commit baa95ca

Please sign in to comment.