Use TQDM to track index compilation progress

[lapp0]

Fixes #810

Changes

Use tqdm progress bar within regex.py's create_fsm_index_end_to_end() so the program doesn't silently hang with no indication of status.

Update pre-commit hook to ensure types-tqdm is installed for mypy type checking.

Example

$ python3 810.py 
Compiling FSM index for all state transitions: 62%|██████████████████           | 146/235 [00:12<00:00, 18.96it/s]

Contents of 810.py:

import interegular
from transformers import AutoTokenizer

from outlines.models.transformers import TransformerTokenizer
from outlines.fsm.regex import (
    make_deterministic_fsm,
    make_byte_level_better_fsm,
    create_fsm_index_tokenizer
)


regex_str = "(?:(?:[0-9](?:(?:_)?[0-9])*(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*|(?:[0-9](?:(?:_)?[0-9])*\\.(?:[0-9](?:(?:_)?[0-9])*)?|\\.[0-9](?:(?:_)?[0-9])*)(?:(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*)?)|[0-9](?:(?:_)?[0-9])*)(?:J|j)|(?:[0-9](?:(?:_)?[0-9])*(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*|(?:[0-9](?:(?:_)?[0-9])*\\.(?:[0-9](?:(?:_)?[0-9])*)?|\\.[0-9](?:(?:_)?[0-9])*)(?:(?:e|E)(?:(?:\\+|\\-))?[0-9](?:(?:_)?[0-9])*)?)|0(?:x|X)(?:(?:_)?(?:[0-9]|[a-f]|[A-F]))+|0(?:b|B)(?:(?:_)?[0-1])+|0(?:o|O)(?:(?:_)?[0-7])+|(?:(?i:([ubf]?r?|r[ubf])('([^\\\\']|.)*?'))|(?i:([ubf]?r?|r[ubf])(\"([^\\\"]|.)*?\")))|(?:(?:\r?\n[\t ]*|#[^\n]*))+|[1-9](?:(?:_)?[0-9])*|\\\\[\t \x0c]*\r?\n|continue|nonlocal|assert|global|import|lambda|return|async|await|break|class|False|match|raise|while|yield|case|from|None|pass|True|with|def|del|for|not|try|if|[^\\W\\d]\\w*|#[^\n]*|[\t \x0c]+|\\.\\.\\.|@|\\{|\\(|\\[|\\-|\\+|\\*|\\~"

regex_pattern = interegular.parse_pattern(regex_str)
# Not reduced, so that there are many states
regex_fsm, _ = make_deterministic_fsm(regex_pattern.to_fsm())
bytes_fsm = make_byte_level_better_fsm(regex_fsm, keep_utf8=True)

num_fsm_states = len(regex_fsm.states)
assert num_fsm_states == 220

num_bytes_fsm_states = len(bytes_fsm.states)
assert num_bytes_fsm_states == 235

tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer = TransformerTokenizer(tokenizer)

states_to_token_subsets, empty_token_ids = create_fsm_index_tokenizer(
    bytes_fsm, tokenizer
)

[rlouf]

Looks great, thank you!