diff --git a/.coveragerc b/.coveragerc index 8a671d36..6253d373 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,5 @@ omit = # omit everything in /usr /usr/local/lib/python* tests/* - setup.py branch = True diff --git a/MANIFEST.in b/MANIFEST.in index 9e21c90d..4df15203 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,9 +3,8 @@ include CONTRIBUTING.md include LICENSE include MANIFEST.in include README.md -include setup.py include version.txt -include osbenchmark/min-os-version.txt +include osbenchmark/min-version.txt recursive-include docs * recursive-include osbenchmark/resources * # recursive-exclude * __pycache__ diff --git a/Makefile b/Makefile index e6c75fa7..61a9661a 100644 --- a/Makefile +++ b/Makefile @@ -70,9 +70,9 @@ python-caches-clean: -@find . -name "__pycache__" -prune -exec rm -rf -- \{\} \; -@find . -name ".pyc" -prune -exec rm -rf -- \{\} \; -# Note: pip will not update project dependencies (specified either in the install_requires or the extras -# section of the setup.py) if any version is already installed; therefore we recommend -# recreating your environments whenever your project dependencies change. +# Note: pip will not update project dependencies (specified in pyproject.toml) if any version is +# already installed; therefore we recommend recreating your environments whenever your project +# dependencies change. tox-env-clean: rm -rf .tox @@ -83,18 +83,18 @@ test: develop pytest tests/ it: pyinst check-java python-caches-clean tox-env-clean - @which tox || $(PIP) install tox + @tox --version 2>/dev/null | grep -qE '^[4-9]\.' || $(PIP) install "tox>=4" tox it312 it313: pyinst check-java python-caches-clean tox-env-clean - @which tox || $(PIP) install tox + @tox --version 2>/dev/null | grep -qE '^[4-9]\.' || $(PIP) install "tox>=4" tox -e $(@:it%=py%) benchmark: pytest benchmarks/ coverage: - coverage run setup.py test + coverage run -m pytest tests/ coverage html release-checks: diff --git a/PYTHON_SUPPORT_GUIDE.md b/PYTHON_SUPPORT_GUIDE.md index e7a5f5cf..e4423fe4 100644 --- a/PYTHON_SUPPORT_GUIDE.md +++ b/PYTHON_SUPPORT_GUIDE.md @@ -11,10 +11,8 @@ Make changes to the following files and open a PR titled * `.ci/variables.json`: Update Python variables and `MIN_PY_VER` as needed. For example: to add Python `3.13`, ensure there is a `PY313` variable set to the latest patch release such as `3.13.1`. -* `.github/workflows/integ-tests.yml`: Update supported Python versions in - the `python-versions` section. -* `setup.py`: Update `supported_python_versions`. -* `tox.ini`: Update `env_list`. +* `pyproject.toml`: Update `requires-python` and the `classifiers` list under `[project]`. +* `pyproject.toml`: Update `envlist` in the `[tool.tox]` `legacy_tox_ini` section. * `Makefile`: If updating the minimum supported Python version, update the `pyinst` target and `check-pip`: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..91cc3800 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "solr-orbit" +dynamic = ["version"] +description = "Macrobenchmarking framework for Apache Solr" +readme = {file = "README.md", content-type = "text/markdown"} +license = {text = "Apache License, Version 2.0"} +requires-python = ">=3.12,<3.14" +classifiers = [ + "Topic :: System :: Benchmark", + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + # License: BSD + # Solr HTTP client for data operations (indexing, search, commit, optimize) + "pysolr>=3.10.0", + # License: Apache 2.0 + # HTTP client for Solr V2 API admin operations + # transitive dependencies: + # urllib3: MIT + "requests>=2.28.0", + # License: BSD + "psutil>=5.8.0", + # License: MIT + "py-cpuinfo>=7.0.0", + # License: MIT + "tabulate>=0.9.0", + # License: MIT + "jsonschema>=3.1.1", + # License: BSD + "Jinja2>=3.1.3", + # License: BSD + "markupsafe>=2.0.1", + # License: MIT + # With 3.10.7, we get InvalidActorAddress exception while initialize Actor + "thespian>=3.10.1,<3.10.7", + # always use the latest version, these are certificate files... + # License: MPL 2.0 + "certifi", + # License: Apache 2.0 + "yappi>=1.4.0", + # License: BSD + "ijson>=2.6.1", + # License: Apache 2.0 + # transitive dependencies: + # google-crc32c: Apache 2.0 + "google-resumable-media>=1.1.0", + # License: Apache 2.0 + "google-auth>=1.22.1", + # License: MIT + "wheel>=0.38.4", + # License: Apache 2.0 + # transitive dependencies: + # botocore: Apache 2.0 + # jmespath: MIT + # s3transfer: Apache 2.0 + "boto3>=1.28.62", + # Licence: BSD-3-Clause + "zstandard>=0.22.0", + # License: BSD + # Required for knnvector workload + "h5py>=3.10.0", + # License: BSD + # Required for knnvector workload + "numpy>=1.24.2,<=1.26.4", + # License: MIT + "tqdm", + # License: MIT + "faker", + # License: BSD + "pandas>=1.4.3", + # License: MIT + "mimesis==11.1.0", + # Licence: BSD-3-Clause + "dask", + # Licence: BSD-3-Clause + "dask[distributed]", + # Licence: BSD-3-Clause + "bokeh!=3.0.*,>=2.4.2", + # License: MIT + "pydantic>=2.10.6", + # License: MIT + "pydantic_core>=2.27.2", + # License: MIT + "PyYAML>=5.4", +] + +[project.urls] +Homepage = "https://github.com/apache/solr-orbit" + +[project.scripts] +solr-orbit = "osbenchmark.benchmark:main" +solr-orbitd = "osbenchmark.benchmarkd:main" + +[project.optional-dependencies] +test = [ + "ujson", + "pytest==7.2.2", + # Upgraded from 3.2.2: py.io.TerminalWriter was removed in py 1.11+, + # making 3.x incompatible with Python 3.12. 4.0+ dropped the py dependency. + "pytest-benchmark>=4.0.0", + "pytest-asyncio==0.14.0", +] +develop = [ + "ujson", + "pytest==7.2.2", + # Upgraded from 3.2.2: py.io.TerminalWriter was removed in py 1.11+, + # making 3.x incompatible with Python 3.12. 4.0+ dropped the py dependency. + "pytest-benchmark>=4.0.0", + "pytest-asyncio==0.14.0", + "tox>=4.0", + "coverage==5.5", + "twine==6.0.1", + "wheel>=0.38.4", + "github3.py==1.3.0", + "pylint==3.3.9", +] + +# --------------------------------------------------------------------------- +# Setuptools configuration +# --------------------------------------------------------------------------- + +[tool.setuptools] +include-package-data = true +script-files = ["scripts/expand-data-corpus.py"] + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests*", "benchmarks*", "it*"] + +[tool.setuptools.package-data] +"*" = ["*.json", "*.yml"] + +[tool.setuptools.dynamic] +version = {file = "version.txt"} + +# --------------------------------------------------------------------------- +# pytest +# --------------------------------------------------------------------------- + +[tool.pytest.ini_options] +log_cli = false +log_level = "INFO" +addopts = "--verbose --color=yes" +testpaths = ["tests"] +junit_family = "xunit2" +junit_logging = "all" + +# --------------------------------------------------------------------------- +# tox +# --------------------------------------------------------------------------- + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = + py312,py313 +platform = + linux|darwin + +[testenv] +deps= + pytest +passenv = + HOME + JAVA*_HOME + BENCHMARK_HOME + SSH_AUTH_SOCK + THESPLOG_FILE + THESPLOG_FILE_MAXSIZE + THESPLOG_THRESHOLD +# we do not pass LANG and LC_ALL anymore in order to isolate integration tests +# from the test environment. OSBenchmark needs to enforce UTF-8 encoding in every +# place so we intentionally set LC_ALL to C. +setenv = + LC_ALL=C + # According to http://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html + # LC_ALL should have priority but to ensure that non-confirming + # applications behave identically, we also set LANG explicitly. + LANG=C +commands = + python -V + pytest -s it --junitxml=junit-{envname}-it.xml + +allowlist_externals = + pytest +""" diff --git a/run.sh b/run.sh index f93f3c6e..23fa7272 100755 --- a/run.sh +++ b/run.sh @@ -28,15 +28,14 @@ readonly BINARY_NAME="${__BENCHMARK_INTERNAL_BINARY_NAME}" readonly HUMAN_NAME="${__BENCHMARK_INTERNAL_HUMAN_NAME}" -install_osbenchmark_with_setuptools () { +install_osbenchmark () { # Check if optional parameter with benchmark binary path, points to an existing executable file. if [[ $# -ge 1 && -n $1 ]]; then if [[ -f $1 && -x $1 ]]; then return; fi fi + # Workaround system pip conflicts, https://github.com/pypa/pip/issues/5599 if [[ ${IN_VIRTUALENV} == 0 ]]; then - # https://setuptools.readthedocs.io/en/latest/setuptools.html suggests not invoking setup.py directly - # Also workaround system pip conflicts, https://github.com/pypa/pip/issues/5599 python3 -m pip install --quiet --user --upgrade --editable .[develop] else python3 -m pip install --quiet --upgrade --editable .[develop] @@ -99,7 +98,7 @@ then then echo "Auto-updating solr-orbit from ${REMOTE}" git rebase ${REMOTE}/master --quiet - install_osbenchmark_with_setuptools + install_osbenchmark #else # offline - skipping update fi @@ -125,14 +124,14 @@ if [[ $IN_VIRTUALENV == 0 ]] then BENCHMARK_ROOT=$(python3 -c "import site; print(site.USER_BASE)") BENCHMARK_BIN=${BENCHMARK_ROOT}/bin/${BINARY_NAME} - install_osbenchmark_with_setuptools "${BENCHMARK_BIN}" + install_osbenchmark "${BENCHMARK_BIN}" if [[ -x $BENCHMARK_BIN ]]; then ${BENCHMARK_BIN} "$@" else echo "Cannot execute ${HUMAN_NAME} in ${BENCHMARK_BIN}." fi else - install_osbenchmark_with_setuptools "${BINARY_NAME}" + install_osbenchmark "${BINARY_NAME}" ${BINARY_NAME} "$@" fi diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d4e57b23..00000000 --- a/setup.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[aliases] -test=pytest - -[tool:pytest] -# set to true for more verbose output of tests -log_cli=false -log_level=INFO -addopts = --verbose --color=yes -testpaths = tests -junit_family = xunit2 -junit_logging = all diff --git a/setup.py b/setup.py deleted file mode 100644 index bd332d69..00000000 --- a/setup.py +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# Modifications by Apache Solr contributors; see git log for details. -# Licensed under the Apache License, Version 2.0. -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from os.path import join, dirname - -try: - from setuptools import setup, find_packages -except ImportError: - print("*** Could not find setuptools. Did you install pip3? *** \n\n") - raise - - -def str_from_file(name): - with open(join(dirname(__file__), name)) as f: - return f.read().strip() - - -raw_version = str_from_file("version.txt") -VERSION = raw_version.split(".") -__version__ = VERSION -__versionstr__ = raw_version - -long_description = str_from_file("README.md") - -# tuples of (major, minor) of supported Python versions ordered from lowest to highest -supported_python_versions = [(3, 12), (3, 13)] - -################################################################################################ -# -# Adapt `create-notice.sh` whenever changing dependencies here. Also, rerun -# `pip install -e .` after changing dependencies to ensure that all -# dependencies are up to date when testing in the virtual environment -# -# That script grabs all license files so we include them in the notice file. -# -################################################################################################ -install_requires = [ - # License: BSD - # Solr HTTP client for data operations (indexing, search, commit, optimize) - "pysolr>=3.10.0", - # License: Apache 2.0 - # HTTP client for Solr V2 API admin operations - # transitive dependencies: - # urllib3: MIT - "requests>=2.28.0", - # License: BSD - "psutil>=5.8.0", - # License: MIT - "py-cpuinfo>=7.0.0", - # License: MIT - "tabulate>=0.9.0", - # License: MIT - "jsonschema>=3.1.1", - # License: BSD - "Jinja2>=3.1.3", - # License: BSD - "markupsafe>=2.0.1", - # License: MIT - # With 3.10.7, we get InvalidActorAddress exception while initialize Actor - "thespian>=3.10.1,<3.10.7", - # recommended library for thespian to identify actors more easily with `ps` - # "setproctitle==1.1.10", - # always use the latest version, these are certificate files... - # License: MPL 2.0 - "certifi", - # License: Apache 2.0 - "yappi>=1.4.0", - # License: BSD - "ijson>=2.6.1", - # License: Apache 2.0 - # transitive dependencies: - # google-crc32c: Apache 2.0 - "google-resumable-media>=1.1.0", - # License: Apache 2.0 - "google-auth>=1.22.1", - # License: MIT - "wheel>=0.38.4", - # License: Apache 2.0 - # transitive dependencies: - # botocore: Apache 2.0 - # jmespath: MIT - # s3transfer: Apache 2.0 - "boto3>=1.28.62", - # Licence: BSD-3-Clause - "zstandard>=0.22.0", - # License: BSD - # Required for knnvector workload - "h5py>=3.10.0", - # License: BSD - # Required for knnvector workload - "numpy>=1.24.2,<=1.26.4", - # License: MIT - "tqdm", - # License: MIT - "faker", - # License: BSD - "pandas>=1.4.3", - # License: MIT - "mimesis==11.1.0", - # Licence: BSD-3-Clause - "dask", - # Licence: BSD-3-Clause - "dask[distributed]", - # Licence: BSD-3-Clause - "bokeh!=3.0.*,>=2.4.2", - # License: MIT - "pydantic>=2.10.6", - # License: MIT - "pydantic_core>=2.27.2", - # License: MIT - "PyYAML>=5.4", -] - -tests_require = [ - "ujson", - "pytest==7.2.2", - "pytest-benchmark==3.2.2", - "pytest-asyncio==0.14.0" -] - -# These packages are only required when developing solr-orbit -develop_require = [ - "tox==3.14.0", - "coverage==5.5", - "twine==6.0.1", - "wheel>=0.38.4", - "github3.py==1.3.0", - "pylint==3.3.9" -] - -python_version_classifiers = ["Programming Language :: Python :: {}.{}".format(major, minor) - for major, minor in supported_python_versions] - -first_supported_version = "{}.{}".format(supported_python_versions[0][0], supported_python_versions[0][1]) -# next minor after the latest supported version -first_unsupported_version = "{}.{}".format(supported_python_versions[-1][0], supported_python_versions[-1][1] + 1) - -setup(name="solr-orbit", - version=__versionstr__, - description="Macrobenchmarking framework for Apache Solr", - long_description=long_description, - long_description_content_type='text/markdown', - url="https://github.com/apache/solr-orbit", - license="Apache License, Version 2.0", - packages=find_packages( - where=".", - exclude=("tests*", "benchmarks*", "it*") - ), - include_package_data=True, - # supported Python versions. This will prohibit pip (> 9.0.0) from even installing solr-orbit on an unsupported - # Python version. - # See also https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires - # - # According to https://www.python.org/dev/peps/pep-0440/#version-matching, a trailing ".*" should - # ignore patch versions: - # - # "additional trailing segments will be ignored when determining whether or not a version identifier matches - # the clause" - # - # However, with the pattern ">=3.5.*,<=3.8.*", the version "3.8.0" is not accepted. Therefore, we match - # the minor version after the last supported one (i.e. if 3.8 is the last supported, we'll emit "<3.9") - python_requires=">={},<{}".format(first_supported_version, first_unsupported_version), - package_data={"": ["*.json", "*.yml"]}, - install_requires=install_requires, - test_suite="tests", - tests_require=tests_require, - extras_require={ - "develop": tests_require + develop_require - }, - entry_points={ - "console_scripts": [ - "solr-orbit=osbenchmark.benchmark:main", - "solr-orbitd=osbenchmark.benchmarkd:main", - ], - }, - scripts=['scripts/expand-data-corpus.py'], - classifiers=[ - "Topic :: System :: Benchmark", - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Intended Audience :: Developers", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - ] + python_version_classifiers, - zip_safe=False) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 5a35ca92..00000000 --- a/tox.ini +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################### -# -# tox configuration for Apache Solr Orbit. -# -# Invocation: Run `make it` -# -############################################################################### -[tox] -envlist = - py312,py313 -platform = - linux|darwin - -[testenv] -deps= - pytest -passenv = - HOME - JAVA*_HOME - BENCHMARK_HOME - SSH_AUTH_SOCK - THESPLOG_FILE - THESPLOG_FILE_MAXSIZE - THESPLOG_THRESHOLD -# we do not pass LANG and LC_ALL anymore in order to isolate integration tests -# from the test environment. OSBenchmark needs to enforce UTF-8 encoding in every -# place so we intentionally set LC_ALL to C. -setenv = - LC_ALL=C - # According to http://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html - # LC_ALL should have priority but to ensure that non-confirming - # applications behave identically, we also set LANG explicitly. - LANG=C -commands = - python -V - pytest -s it --junitxml=junit-{envname}-it.xml - -whitelist_externals = - pytest