up tokenizers deps

smallcloudai · Apr 19, 2024 · 7e59797 · 7e59797
1 parent 7df3301
commit 7e59797
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -28,15 +28,15 @@ class PyPackage:
         data=["refact_toolbox_db/htmls/*.html"]),
     "refact_utils": PyPackage(),
     "refact_encoding": PyPackage(
-        requires=["tiktoken", "tokenizers==0.15.0", "sentencepiece", "termcolor", "protobuf"],
+        requires=["tiktoken", "tokenizers>=0.15.0", "sentencepiece", "termcolor", "protobuf"],
         data=["*.json"]),
     "refact_scratchpads": PyPackage(
         requires=["termcolor", "torch"],
         requires_packages=["refact_encoding", "code_contrast", "refact_scratchpads_no_gpu"]),
     "refact_scratchpads_no_gpu": PyPackage(
         requires=["termcolor", "aiohttp", "tiktoken", "openai>=1.0.0", "ujson", "setproctitle"]),
     "refact_data_pipeline": PyPackage(
-        requires=["numpy", "tokenizers==0.15.0", "torch", "requests>=2.31.0", "cloudpickle", "blobfile",
+        requires=["numpy", "tokenizers>=0.15.0", "torch", "requests>=2.31.0", "cloudpickle", "blobfile",
                   "tqdm", "dataclasses_json", "termcolor", 'more_itertools', "cdifflib",
                   "ujson", "zstandard", "scipy", "einops", "matplotlib", "giturlparse",
                   "jsonlines", "binpacking", "filelock", "tables==3.8.0", "pygments", "kshingle"],