Optimize PygmentsLexer creation

[nitanmarcel]

I have an editor and I'm using PygmentsLexer to lex my file but this ends up with slow start speeds for my application. I'm using asyncio and each time I start my application with the layout attached the app takes 1 second to start, with 0.30 seconds taken by the PygmentsLexer.

What can I do to optimize this? Maybe there's a way to create the lexer after my app starts?

[nitanmarcel]

Ok, I managed to reduced it by a lot using an already dictionary I had with language ids and getting the pygment lexer by using that.

In the code I actually use importlib instead of using the alias detection function in pygments.

import importlib
from typing import Callable, Dict

from prompt_toolkit.lexers import SimpleLexer, PygmentsLexer, Lexer
from prompt_toolkit.document import Document
from prompt_toolkit.formatted_text.base import StyleAndTextTuples

from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound

_CACHE: Dict[str, Lexer] = {}


class FileLexer(Lexer):
    def __init__(self, editor, path: str) -> None:
        self._editor = editor
        self._path = path

    def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]:
        filetype = self._editor.filetype.guess_filetype(self._path, document.text)
        if filetype not in _CACHE.keys():
            known = _KNOWN_LEXERS.get(filetype, None)
            if known is not None:
                module, cls = known
                module = importlib.import_module(module)
                cls = getattr(module, cls)
                _CACHE[filetype] = PygmentsLexer(cls, sync_from_start=False)
            else:
                try:
                    _CACHE[filetype] = PygmentsLexer(
                        get_lexer_by_name(filetype).__class__
                    )
                except ClassNotFound as _:
                    _CACHE[filetype] = SimpleLexer()
        return _CACHE[filetype].lex_document(document)


_KNOWN_LEXERS = {
    "ini": ("pygments.lexers.configs", "IniLexer"),
    "cpp": ("pygments.lexers.c_cpp", "CppLexer"),
    "c": ("pygments.lexers.c_cpp", "CLexer"),
    "objc": ("pygments.lexers.objective", "ObjectiveCLexer"),
    "ocaml": ("pygments.lexers.ml", "OcamlLexer"),
    "sass": ("pygments.lexers.css", "SassLexer"),
    "zig": ("pygments.lexers.zig", "ZigLexer"),
    "gdscript": ("pygments.lexers.gdscript", "GDScriptLexer"),
    "go": ("pygments.lexers.go", "GoLexer"),
    "fsharp": ("pygments.lexers.dotnet", "FSharpLexer"),
    "meson": ("pygments.lexers.meson", "MesonLexer"),
    "ruby": ("pygments.lexers.ruby", "RubyLexer"),
    "r": ("pygments.lexers.r", "SLexer"),
    "tcl": ("pygments.lexers.tcl", "TclLexer"),
    "scss": ("pygments.lexers.css", "ScssLexer"),
    "css": ("pygments.lexers.css", "CssLexer"),
    "nim": ("pygments.lexers.nimrod", "NimrodLexer"),
    "protobuf": ("pygments.lexers.dsls", "ProtoBufLexer"),
    "java": ("pygments.lexers.jvm", "JavaLexer"),
    "erlang": ("pygments.lexers.erlang", "ErlangLexer"),
    "awk": ("pygments.lexers.textedit", "AwkLexer"),
    "coffee": ("pygments.lexers.javascript", "CoffeeScriptLexer"),
    "d": ("pygments.lexers.d", "DLexer"),
    "javascript": ("pygments.lexers.javascript", "JavascriptLexer"),
    "typescript": ("pygments.lexers.javascript", "TypeScriptLexer"),
    "gas": ("pygments.lexers.asm", "GasLexer"),
    "graphql": ("pygments.lexers.graphql", "GraphQLLexer"),
    "sh": ("pygments.lexers.shell", "BashLexer"),
    "ragel": ("pygments.lexers.parsers", "RagelLexer"),
    "html": ("pygments.lexers.html", "HtmlLexer"),
    "xml": ("pygments.lexers.html", "XmlLexer"),
    "python": ("pygments.lexers.python", "PythonLexer"),
    "html+handlebars": ("pygments.lexers.templates", "HandlebarsHtmlLexer"),
    "scala": ("pygments.lexers.jvm", "ScalaLexer"),
    "dhall": None,
    "kickstart": None,
    "swift": ("pygments.lexers.objective", "SwiftLexer"),
    "twig": ("pygments.lexers.templates", "TwigLexer"),
    "coq": ("pygments.lexers.theorem", "CoqLexer"),
    "hare": None,
    "cue": None,
    "groovy": ("pygments.lexers.jvm", "GroovyLexer"),
    "moon": ("pygments.lexers.scripting", "MoonScriptLexer"),
    "elm": ("pygments.lexers.elm", "ElmLexer"),
    "glep42": None,
    "diff": ("pygments.lexers.diff", "DiffLexer"),
    "toml": ("pygments.lexers.configs", "TOMLLexer"),
    "pascal": ("pygments.lexers.pascal", "DelphiLexer"),
    "delphi": ("pygments.lexers.pascal", "DelphiLexer"),
    "haskell": ("pygments.lexers.haskell", "HaskellLexer"),
    "julia": ("pygments.lexers.julia", "JuliaLexer"),
    "justfile": None,
    "lua": ("pygments.lexers.scripting", "LuaLexer"),
    "ledger": None,
    "gleam": None,
    "prolog": ("pygments.lexers.prolog", "PrologLexer"),
    "php": ("pygments.lexers.php", "PhpLexer"),
    "makefile": ("pygments.lexers.make", "MakefileLexer"),
    "crystal": ("pygments.lexers.crystal", "CrystalLexer"),
    "yaml": ("pygments.lexers.data", "YamlLexer"),
    "sml": ("pygments.lexers.ml", "SMLLexer"),
    "clojure": ("pygments.lexers.jvm", "ClojureLexer"),
    "taskpaper": None,
    "scheme": ("pygments.lexers.lisp", "SchemeLexer"),
    "pony": ("pygments.lexers.pony", "PonyLexer"),
    "sql": ("pygments.lexers.sql", "SqlLexer"),
    "markdown": ("pygments.lexers.markup", "MarkdownLexer"),
    "haml": ("pygments.lexers.html", "HamlLexer"),
    "asciidoc": None,
    "cucumber": ("pygments.lexers.testing", "GherkinLexer"),
    "janet": ("pygments.lexers.lisp", "JanetLexer"),
    "lisp": ("pygments.lexers.lisp", "CommonLispLexer"),
    "rust": ("pygments.lexers.rust", "RustLexer"),
    "dart": ("pygments.lexers.javascript", "DartLexer"),
    "nix": ("pygments.lexers.nix", "NixLexer"),
    "perl": ("pygments.lexers.perl", "PerlLexer"),
    "mercury": None,
    "elvish": None,
    "kotlin": ("pygments.lexers.jvm", "KotlinLexer"),
    "cmake": ("pygments.lexers.make", "CMakeLexer"),
    "odin": ("pygments.lexers.archetype", "OdinLexer"),
    "tupfile": None,
    "man": ("pygments.lexers.markup", "GroffLexer"),
    "dockerfile": ("pygments.lexers.configs", "DockerLexer"),
    "latex": ("pygments.lexers.markup", "TexLexer"),
    "typst": ("pygments.lexers.typst", "TypstLexer"),
    "fidl": None,
    "fish": ("pygments.lexers.shell", "FishShellLexer"),
    "elixir": ("pygments.lexers.erlang", "ElixirLexer"),
    "eex": None,
    "fennel": ("pygments.lexers.lisp", "FennelLexer"),
    "mlb": None,
    "ninja": None,
    "json": ("pygments.lexers.data", "JsonLexer"),
    "pug": ("pygments.lexers.html", "PugLexer"),
    "terraform": ("pygments.lexers.configs", "TerraformLexer"),
    "restructuredtext": ("pygments.lexers.markup", "RstLexer"),
    "xnosh": None,
}