From 6810c9aba88541432cc98207e07eca0f5554480f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 17 May 2021 09:42:54 +0300 Subject: [PATCH 1/5] feat(lexers): add ASCII armored --- AUTHORS | 1 + pygments/lexers/_mapping.py | 1 + pygments/lexers/asc.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 pygments/lexers/asc.py diff --git a/AUTHORS b/AUTHORS index e8df18588f..634935aa8e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -201,6 +201,7 @@ Other contributors, listed alphabetically, are: * Robert Simmons -- Standard ML lexer * Kirill Simonov -- YAML lexer * Corbin Simpson -- Monte lexer +* Ville Skyttä -- ASCII armored lexer * Alexander Smishlajev -- Visual FoxPro lexer * Steve Spigarelli -- XQuery lexer * Jerome St-Louis -- eC lexer diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index aab8bde441..c862759215 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -41,6 +41,7 @@ 'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()), 'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)), 'ArrowLexer': ('pygments.lexers.arrow', 'Arrow', ('arrow',), ('*.arw',), ()), + 'AscLexer': ('pygments.lexers.asc', 'ASCII armored', ('asc', 'pem'), ('*.asc', '*.pem', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', 'id_ed25519_sk', 'id_rsa'), (['application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature'],)), 'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)), 'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asymptote', 'asy'), ('*.asy',), ('text/x-asymptote',)), 'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug',), ()), diff --git a/pygments/lexers/asc.py b/pygments/lexers/asc.py new file mode 100644 index 0000000000..c96a6d3824 --- /dev/null +++ b/pygments/lexers/asc.py @@ -0,0 +1,49 @@ +""" + pygments.lexers.asc + ~~~~~~~~~~~~~~~~~~~ + + Lexer for various ASCII armored files. + + :copyright: Copyright 2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import re + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Generic, Name, Operator, Text, String + +__all__ = ['AscLexer'] + + +class AscLexer(RegexLexer): + """ + Lexer for ASCII armored files, containing `-----BEGIN/END ...-----` wrapped base64 data. + """ + name = 'ASCII armored' + aliases = ['asc', 'pem'] + filenames = [ + '*.asc', # PGP; *.gpg, *.pgp, and *.sig too, but those can be binary + '*.pem', # X.509; *.cer, *.crt, *.csr, and key etc too, but those can be binary + 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', 'id_ed25519_sk', 'id_rsa', # SSH private keys + ] + mimetypes = ['application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature'], + + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'\s+', Text), + (r'^-----BEGIN [^\n]+-----$', Generic.Heading, 'data'), + (r'\S+', Comment), + ], + 'data': [ + (r'\s+', Text), + (r'^([^:]+)(:)([ \t]+)(.*)', bygroups(Name.Attribute, Operator, Text, String)), + (r'^-----END [^\n]+-----$', Generic.Heading, 'root'), + (r'\S+', String), + ], + } + + def analyse_text(text): + if re.search(r'^-----BEGIN [^\n]+-----\r?\n', text): + return True From 35643bdcbd591c26fe5560cde8a1cba229f4f8a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 17 May 2021 20:09:56 +0300 Subject: [PATCH 2/5] docs(AscLexer): add versionadded --- pygments/lexers/asc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pygments/lexers/asc.py b/pygments/lexers/asc.py index c96a6d3824..908c0b67f4 100644 --- a/pygments/lexers/asc.py +++ b/pygments/lexers/asc.py @@ -18,6 +18,8 @@ class AscLexer(RegexLexer): """ Lexer for ASCII armored files, containing `-----BEGIN/END ...-----` wrapped base64 data. + + .. versionadded:: 2.10 """ name = 'ASCII armored' aliases = ['asc', 'pem'] From dc03ea1a26e6746547c47197ce4b6fde44ae40d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 17 May 2021 20:40:40 +0300 Subject: [PATCH 3/5] test(asc): add id_ecdsa example file --- tests/examplefiles/asc/id_ecdsa | 9 ++++++++ tests/examplefiles/asc/id_ecdsa.output | 29 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/examplefiles/asc/id_ecdsa create mode 100644 tests/examplefiles/asc/id_ecdsa.output diff --git a/tests/examplefiles/asc/id_ecdsa b/tests/examplefiles/asc/id_ecdsa new file mode 100644 index 0000000000..b9245e829c --- /dev/null +++ b/tests/examplefiles/asc/id_ecdsa @@ -0,0 +1,9 @@ +-----BEGIN EC PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,IVIVIVIVIVIVIVIVIVIVIVIVIVIVIVIV + +ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ +ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ +ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ +ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq= +-----END EC PRIVATE KEY----- diff --git a/tests/examplefiles/asc/id_ecdsa.output b/tests/examplefiles/asc/id_ecdsa.output new file mode 100644 index 0000000000..e9a3b003a0 --- /dev/null +++ b/tests/examplefiles/asc/id_ecdsa.output @@ -0,0 +1,29 @@ +'-----BEGIN EC PRIVATE KEY-----' Generic.Heading +'\n' Text + +'Proc-Type' Name.Attribute +':' Operator +' ' Text +'4,ENCRYPTED' Literal.String +'\n' Text + +'DEK-Info' Name.Attribute +':' Operator +' ' Text +'AES-128-CBC,IVIVIVIVIVIVIVIVIVIVIVIVIVIVIVIV' Literal.String +'\n\n' Text + +'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String +'\n' Text + +'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String +'\n' Text + +'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String +'\n' Text + +'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq=' Literal.String +'\n' Text + +'-----END EC PRIVATE KEY-----' Generic.Heading +'\n' Text From 2755ceef3d4f4e102c58e195fc7b10bb3c04ed53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Mon, 17 May 2021 20:41:17 +0300 Subject: [PATCH 4/5] fix(AscLexer): mimetypes stray trailing comma --- pygments/lexers/_mapping.py | 2 +- pygments/lexers/asc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index c862759215..11f4de4153 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -41,7 +41,7 @@ 'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()), 'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)), 'ArrowLexer': ('pygments.lexers.arrow', 'Arrow', ('arrow',), ('*.arw',), ()), - 'AscLexer': ('pygments.lexers.asc', 'ASCII armored', ('asc', 'pem'), ('*.asc', '*.pem', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', 'id_ed25519_sk', 'id_rsa'), (['application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature'],)), + 'AscLexer': ('pygments.lexers.asc', 'ASCII armored', ('asc', 'pem'), ('*.asc', '*.pem', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', 'id_ed25519_sk', 'id_rsa'), ('application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature')), 'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)), 'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asymptote', 'asy'), ('*.asy',), ('text/x-asymptote',)), 'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug',), ()), diff --git a/pygments/lexers/asc.py b/pygments/lexers/asc.py index 908c0b67f4..471c13a0e0 100644 --- a/pygments/lexers/asc.py +++ b/pygments/lexers/asc.py @@ -28,7 +28,7 @@ class AscLexer(RegexLexer): '*.pem', # X.509; *.cer, *.crt, *.csr, and key etc too, but those can be binary 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', 'id_ed25519_sk', 'id_rsa', # SSH private keys ] - mimetypes = ['application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature'], + mimetypes = ['application/pgp-keys', 'application/pgp-encrypted', 'application/pgp-signature'] flags = re.MULTILINE From 6e60767915b2dd8d662a5304ce9192a33097aba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Tue, 18 May 2021 16:38:11 +0300 Subject: [PATCH 5/5] fix(asc): use Whitespace instead of Text for whitespace --- pygments/lexers/asc.py | 8 ++++---- tests/examplefiles/asc/id_ecdsa.output | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pygments/lexers/asc.py b/pygments/lexers/asc.py index 471c13a0e0..f775fd0791 100644 --- a/pygments/lexers/asc.py +++ b/pygments/lexers/asc.py @@ -10,7 +10,7 @@ import re from pygments.lexer import RegexLexer, bygroups -from pygments.token import Comment, Generic, Name, Operator, Text, String +from pygments.token import Comment, Generic, Name, Operator, String, Whitespace __all__ = ['AscLexer'] @@ -34,13 +34,13 @@ class AscLexer(RegexLexer): tokens = { 'root': [ - (r'\s+', Text), + (r'\s+', Whitespace), (r'^-----BEGIN [^\n]+-----$', Generic.Heading, 'data'), (r'\S+', Comment), ], 'data': [ - (r'\s+', Text), - (r'^([^:]+)(:)([ \t]+)(.*)', bygroups(Name.Attribute, Operator, Text, String)), + (r'\s+', Whitespace), + (r'^([^:]+)(:)([ \t]+)(.*)', bygroups(Name.Attribute, Operator, Whitespace, String)), (r'^-----END [^\n]+-----$', Generic.Heading, 'root'), (r'\S+', String), ], diff --git a/tests/examplefiles/asc/id_ecdsa.output b/tests/examplefiles/asc/id_ecdsa.output index e9a3b003a0..0c3ad48b3f 100644 --- a/tests/examplefiles/asc/id_ecdsa.output +++ b/tests/examplefiles/asc/id_ecdsa.output @@ -1,29 +1,29 @@ '-----BEGIN EC PRIVATE KEY-----' Generic.Heading -'\n' Text +'\n' Text.Whitespace 'Proc-Type' Name.Attribute ':' Operator -' ' Text +' ' Text.Whitespace '4,ENCRYPTED' Literal.String -'\n' Text +'\n' Text.Whitespace 'DEK-Info' Name.Attribute ':' Operator -' ' Text +' ' Text.Whitespace 'AES-128-CBC,IVIVIVIVIVIVIVIVIVIVIVIVIVIVIVIV' Literal.String -'\n\n' Text +'\n\n' Text.Whitespace 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String -'\n' Text +'\n' Text.Whitespace 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String -'\n' Text +'\n' Text.Whitespace 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' Literal.String -'\n' Text +'\n' Text.Whitespace 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopq=' Literal.String -'\n' Text +'\n' Text.Whitespace '-----END EC PRIVATE KEY-----' Generic.Heading -'\n' Text +'\n' Text.Whitespace