Bqn lexer: lexer for the BQN array programming language

pygments/lexers/_mapping.py

@@ -41,6 +41,7 @@
     'BBCBasicLexer': ('pygments.lexers.basic', 'BBC Basic', ('bbcbasic',), ('*.bbc',), ()),
     'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
     'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
+    'BQNLexer': ('pygments.lexers.bqn', 'BQN', ('bqn',), ('*.bqn',), ()),
     'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
     'BareLexer': ('pygments.lexers.bare', 'BARE', ('bare',), ('*.bare',), ()),
     'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),

pygments/lexers/bqn.py

@@ -0,0 +1,109 @@
+"""
+    pygments.lexers.bqn
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for BQN.
+
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['BQNLexer']
+
+
+class BQNLexer(RegexLexer):
+    """
+    A simple BQN lexer.
+    """
+    name = 'BQN'
+    url = 'https://mlochbaum.github.io/BQN/index.html'
+    aliases = ['bqn']
+    filenames = [
+        '*.bqn',
+    ]
+
+    tokens = {
+        'root': [
+            # Whitespace
+            # ==========
+            (r'\s+', Whitespace),
+            #
+            # Comment
+            # =======
+            # '#' is a comment that continues to the end of the line
+            (r'#.*$', Comment.Single),
+            #
+            # Strings
+            # =======
+            (r'\'((\'\')|[^\'])*\'', String.Single),
+            (r'"(("")|[^"])*"', String.Double),
+            #
+            # Null Character
+            # ==============
+            # Literal representation of the null character
+            (r'@', String.Symbol),
+            #
+            # Punctuation
+            # ===========
+            # This token type is used for diamond, commas
+            # and  array and list brackets and strand syntax
+            (r'[◇,\[\]⟨⟩‿]', Punctuation),
+            #
+            # Expression Grouping
+            # ===================
+            # Since this token type is important in BQN, it is not included in
+            # the punctuation token type but rather in the following one
+            (r'[\(\)]', String.Regex), 
+            #
+            # Numbers
+            # =======
+            # Includes the numeric literals and nothing
+            (r'¯?([0-9]+\.?[0-9]+|[0-9]+)([Ee][¯]?[0-9]+)?|¯|∞|π|·', Number),
+            #
+            # Variables
+            # =========
+            (r'\b[a-z]\w*\b', Name.Variable),
+            #
+            # 1-Modifiers
+            # ===========
+            (r'[˙˜˘¨⌜⁼´˝`𝕣]', Name.Attribute),
+            (r'\b_[a-zA-Z0-9]+\b', Name.Attribute),
+            #
+            # 2-Modifiers
+            # ===========
+            (r'[∘○⊸⟜⌾⊘◶⎉⚇⍟⎊]', Name.Property),
+            (r'\b_[a-zA-Z0-9]+_\b', Name.Property),
+            #
+            # Functions
+            # =========
+            # The monadic or dyadic function primatives and function
+            # operands and arguments, along with function self-reference
+            (r'[+\-×÷\*√⌊⌈∧∨¬|≤<>≥=≠≡≢⊣⊢⥊∾≍⋈↑↓↕«»⌽⍉/⍋⍒⊏⊑⊐⊒∊⍷⊔!𝕎𝕏𝔽𝔾𝕊]',
+             Operator),
+            (r'[A-Z]\w*|•\w+\b', Operator),
+            #
+            # Constant
+            # ========
+            (r'˙', Name.Constant),
+            #
+            # Define/Export/Change
+            # ====================
+            (r'[←↩⇐]', Keyword.Declaration),
+            #
+            # Blocks
+            # ======
+            (r'[{}]', Keyword.Type),
+            #
+            # Extra characters
+            # ================
+            (r'[;:?𝕨𝕩𝕗𝕘𝕤]', Name.Entity),
+            #
+
+        ],
+    }
+
+

tests/snippets/bqn/test_arguments.txt

@@ -0,0 +1,28 @@
+---input---
+Ambiv ← { ⟨1,𝕩⟩ ; ⟨2,𝕨,𝕩⟩ }
+
+---tokens---
+'Ambiv'       Operator
+' '           Text.Whitespace
+'←'           Keyword.Declaration
+' '           Text.Whitespace
+'{'           Keyword.Type
+' '           Text.Whitespace
+'⟨'           Punctuation
+'1'           Literal.Number
+','           Punctuation
+'𝕩'           Name.Entity
+'⟩'           Punctuation
+' '           Text.Whitespace
+';'           Name.Entity
+' '           Text.Whitespace
+'⟨'           Punctuation
+'2'           Literal.Number
+','           Punctuation
+'𝕨'           Name.Entity
+','           Punctuation
+'𝕩'           Name.Entity
+'⟩'           Punctuation
+' '           Text.Whitespace
+'}'           Keyword.Type
+'\n'          Text.Whitespace

tests/snippets/bqn/test_comment.txt

@@ -0,0 +1,12 @@
+---input---
+'#' - 1  #This is the comment
+
+---tokens---
+"'#'"         Literal.String.Single
+' '           Text.Whitespace
+'-'           Operator
+' '           Text.Whitespace
+'1'           Literal.Number
+'  '          Text.Whitespace
+'#This is the comment' Comment.Single
+'\n'          Text.Whitespace

tests/snippets/bqn/test_define.txt

@@ -0,0 +1,42 @@
+---input---
+⟨alias⇐a, b⟩ ← {
+  b‿c⇐
+  a⇐2
+  c←÷b↩1+a
+}
+
+---tokens---
+'⟨'           Punctuation
+'alias'       Name.Variable
+'⇐'           Keyword.Declaration
+'a'           Name.Variable
+','           Punctuation
+' '           Text.Whitespace
+'b'           Name.Variable
+'⟩'           Punctuation
+' '           Text.Whitespace
+'←'           Keyword.Declaration
+' '           Text.Whitespace
+'{'           Keyword.Type
+'\n  '        Text.Whitespace
+'b'           Name.Variable
+'‿'           Punctuation
+'c'           Name.Variable
+'⇐'           Keyword.Declaration
+'\n  '        Text.Whitespace
+'a'           Name.Variable
+'⇐'           Keyword.Declaration
+'2'           Literal.Number
+'\n  '        Text.Whitespace
+'c'           Name.Variable
+'←'           Keyword.Declaration
+'÷'           Operator
+'b'           Name.Variable
+'↩'           Keyword.Declaration
+'1'           Literal.Number
+'+'           Operator
+'a'           Name.Variable
+'\n'          Text.Whitespace
+
+'}'           Keyword.Type
+'\n'          Text.Whitespace

tests/snippets/bqn/test_syntax_roles.txt

@@ -0,0 +1,23 @@
+---input---
+⟨ט,√⟩ {𝕎𝕩}⌜ 1‿4‿9
+
+---tokens---
+'⟨'           Punctuation
+'×'           Operator
+'˜'           Name.Attribute
+','           Punctuation
+'√'           Operator
+'⟩'           Punctuation
+' '           Text.Whitespace
+'{'           Keyword.Type
+'𝕎'           Operator
+'𝕩'           Name.Entity
+'}'           Keyword.Type
+'⌜'           Name.Attribute
+' '           Text.Whitespace
+'1'           Literal.Number
+'‿'           Punctuation
+'4'           Literal.Number
+'‿'           Punctuation
+'9'           Literal.Number
+'\n'          Text.Whitespace