diff --git a/requirements/framework.txt b/requirements/framework.txt index 17eac842ac..5f4ec2649d 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -4,12 +4,10 @@ aiohttp attrdict binpacking dacite -datasets<3.0 einops importlib_metadata jieba matplotlib -modelscope[datasets]>=1.17,<1.19 nltk numpy<2.0 oss2 diff --git a/setup.py b/setup.py index 01fd518ddb..3bb117b6be 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,9 @@ import os import shutil from setuptools import find_packages, setup +from typing import List + +from packaging import version def readme(): @@ -115,8 +118,25 @@ def gen_packages_items(): return gen_packages_items() +def add_modelscope_requirement(install_requires: List[str]) -> None: + # The future version will remove. + try: + import modelscope + modelscope_version = modelscope.__version__ + except ImportError: + modelscope_version = '1.18' + + if version.parse(modelscope_version) >= version.parse('1.19'): + install_requires.append('datasets>=3.0') + install_requires.append('modelscope[datasets]>=1.19') + else: + install_requires.append('datasets<3.0') + install_requires.append('modelscope[datasets]>=1.17,<1.19') + + if __name__ == '__main__': install_requires, deps_link = parse_requirements('requirements.txt') + add_modelscope_requirement(install_requires) extra_requires = {} all_requires = [] extra_requires['llm'], _ = parse_requirements('requirements/llm.txt')