-
Notifications
You must be signed in to change notification settings - Fork 12
/
pyproject.toml
110 lines (94 loc) · 3.63 KB
/
pyproject.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
[project]
name = "openvino-tokenizers"
version = "2024.3.0.0"
description = "Convert tokenizers into OpenVINO models"
requires-python = ">=3.8"
readme = {file = "README.md", content-type="text/markdown"}
license = {text = "OSI Approved :: Apache Software License"}
authors = [
{ name = "OpenVINO Developers", email = "openvino@intel.com" },
]
classifiers = [
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
]
dependencies = [
# support of nightly openvino packages with dev suffix
"openvino~=2024.3.0.0.dev"
]
[project.optional-dependencies]
transformers = [
"transformers[sentencepiece] >= 4.36.0",
"tiktoken"
]
# chatglm2 custom tokenizer file imports torch, have to add torch dependency for tests
torch = [
'torch'
]
dev = [
"ruff",
"bandit",
"pytest",
"pytest_harvest",
"pandas",
"openvino_tokenizers[transformers, torch]"
]
benchmark = [
"pandas",
"seaborn",
"tqdm",
"openvino_tokenizers[transformers]"
]
# don't include fuzzing to avoid windows CI issues
fuzzing = [
"atheris",
"openvino_tokenizers[transformers]"
]
all = [
"openvino_tokenizers[dev, transformers]"
]
[project.scripts]
convert_tokenizer = "openvino_tokenizers.cli:convert_hf_tokenizer"
[tool.ruff]
line-length = 119
[tool.ruff.lint]
ignore = ["C901", "E501", "E741", "W605"]
select = ["C", "E", "F", "I", "W"]
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"]
"openvino_tokenizers/hf_parser.py" = ["F821"]
[tool.ruff.lint.isort]
lines-after-imports = 2
[tool.bandit]
tests = ["B301", "B302", "B303", "B304", "B305", "B306", "B308", "B310", "B311", "B312", "B313", "B314", "B315", "B316", "B317", "B318", "B319", "B320", "B321", "B323", "B324", "B401", "B402", "B403", "B404", "B405", "B406", "B407", "B408", "B409", "B410", "B411", "B412", "B413"]
skips = ["B101", "B102", "B103", "B104", "B105", "B106", "B107", "B108", "B110", "B112", "B201", "B501", "B502", "B503", "B504", "B505", "B506", "B507", "B601", "B602", "B603", "B604", "B605", "B606", "B607", "B608", "B609", "B610", "B611", "B701", "B702", "B703"]
no_shell = ["os.execl", "os.execle", "os.execlp", "os.execlpe", "os.execv", "os.execve", "os.execvp", "os.execvpe", "os.spawnl", "os.spawnle", "os.spawnlp", "os.spawnlpe", "os.spawnv", "os.spawnve", "os.spawnvp", "os.spawnvpe", "os.startfile"]
shell = ["os.system", "os.popen", "os.popen2", "os.popen3", "os.popen4", "popen2.popen2", "popen2.popen3", "popen2.popen4", "popen2.Popen3", "popen2.Popen4", "commands.getoutput", "commands.getstatusoutput"]
subprocess = ["subprocess.Popen", "subprocess.call", "subprocess.check_call", "subprocess.check_output", "subprocess.run"]
exclude_dirs = ["tests"]
[tool.py-build-cmake.module]
directory = "python"
[tool.py-build-cmake.sdist]
exclude = ["dist", "tests", "examples", "python/tests"]
[tool.py-build-cmake.cmake]
minimum_version = "3.15"
build_type = "Release"
config = ["Release"]
find_python3 = true
build_args = ["--parallel", "--target", "openvino_tokenizers"]
install_args = ["--strip"]
install_components = ["openvino_tokenizers", "openvino_tokenizers_python", "openvino_tokenizers_licenses"]
[tool.py-build-cmake.cmake.options]
OPENVINO_TOKENIZERS_INSTALL_BINDIR = "openvino_tokenizers/lib"
OPENVINO_TOKENIZERS_INSTALL_LIBDIR = "openvino_tokenizers/lib"
[tool.py-build-cmake.wheel]
python_abi = "none"
[build-system]
requires = [
"py-build-cmake@git+https://github.com/tttapa/py-build-cmake@7ab73da351c7140f06d727a8705bece4cf544cd9",
"cmake~=3.15"
]
build-backend = "py_build_cmake.build"