bbpe: Add token LRU cache and avoid string lookups #246

	name: Test

	on: [push, pull_request, workflow_call]

	jobs:
	test:
	name: Run tests
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ubuntu-latest, macos-latest, windows-latest]
	python-version: ["3.8", "3.12"]
	steps:
	- uses: actions/checkout@v1
	with:
	submodules: true

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	cache: "pip" # caching pip dependencies

	- name: Install requirements
	run: \|
	python -m pip install --upgrade pip setuptools wheel build
	python -m pip install -r requirements.txt

	- name: Build sdist
	run: \|
	python -m build --sdist

	- name: Run mypy
	run: mypy curated_tokenizers

	- name: Delete source directory
	run: \|
	rm -rf curated_tokenizers
	shell: bash

	- name: Uninstall all packages
	run: \|
	python -m pip freeze
	python -m pip freeze --exclude pywin32 > installed.txt
	python -m pip uninstall -y -r installed.txt

	- name: Install from sdist
	run: \|
	SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
	python -m pip install dist/$SDIST
	shell: bash

	- name: Install test dependencies
	run: python -m pip install -r requirements.txt

	- name: Run pytest
	run: python -m pytest --pyargs curated_tokenizers

Provide feedback