diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index cb7ca0e844abb..16501a254f280 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -173,17 +173,10 @@ endif() add_custom_target(paddle_python ALL DEPENDS ${PADDLE_PYTHON_BUILD_DIR}/.timestamp) + if(BUILD_WHL_PACKAGE AND NOT WITH_SETUP_INSTALL) - add_custom_target( - paddle_copy ALL - # generate tensor.pyi for type hints - COMMAND - ${CMAKE_COMMAND} -E env PYTHONPATH=${PADDLE_BINARY_DIR}/python - ${PYTHON_EXECUTABLE} ${PADDLE_SOURCE_DIR}/tools/gen_tensor_stub.py - --input-file - ${PADDLE_SOURCE_DIR}/python/paddle/tensor/tensor.prototype.pyi - --output-file ${PADDLE_BINARY_DIR}/python/paddle/tensor/tensor.pyi - DEPENDS ${PADDLE_PYTHON_BUILD_DIR}/.timestamp_wheel) + add_custom_target(paddle_copy ALL + DEPENDS ${PADDLE_PYTHON_BUILD_DIR}/.timestamp_wheel) add_dependencies(paddle_copy paddle_python) endif() diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index debe183844d66..11c38195e34e8 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -14,12 +14,16 @@ import typing +__is_metainfo_generated = False try: from paddle.cuda_env import * # noqa: F403 from paddle.version import ( # noqa: F401 commit as __git_commit__, full_version as __version__, ) + + __is_metainfo_generated = True + except ImportError: import sys @@ -577,8 +581,7 @@ if os.path.exists(cuh_file): os.environ.setdefault('runtime_include_dir', runtime_include_dir) - -if is_compiled_with_cuda(): +if __is_metainfo_generated and is_compiled_with_cuda(): import os import platform diff --git a/python/paddle/base/framework.py b/python/paddle/base/framework.py index 1c5c7f62b8dbb..4b62b57f4e806 100644 --- a/python/paddle/base/framework.py +++ b/python/paddle/base/framework.py @@ -33,7 +33,6 @@ import numpy as np import paddle -import paddle.version as paddle_version from .. import pir from . import core, unique_name @@ -573,10 +572,10 @@ def require_version(min_version, max_version=None): ) version_installed = [ - paddle_version.major, - paddle_version.minor, - paddle_version.patch, - paddle_version.rc, + paddle.version.major, + paddle.version.minor, + paddle.version.patch, + paddle.version.rc, ] zero_version = ["0", "0", "0", "0"] @@ -591,13 +590,13 @@ def version_cmp(ver_a, ver_b): if version_cmp(version_installed, zero_version) == 0: if max_version is not None: warnings.warn( - f"PaddlePaddle version in [{min_version}, {max_version}] required, but {paddle_version.full_version} installed. " + f"PaddlePaddle version in [{min_version}, {max_version}] required, but {paddle.version.full_version} installed. " "Maybe you are using a develop version, " "please make sure the version is good with your code." ) else: warnings.warn( - f"PaddlePaddle version {min_version} or higher is required, but {paddle_version.full_version} installed, " + f"PaddlePaddle version {min_version} or higher is required, but {paddle.version.full_version} installed, " "Maybe you are using a develop version, " "please make sure the version is good with your code." ) @@ -619,12 +618,12 @@ def version_cmp(ver_a, ver_b): or version_cmp(version_installed, min_version_to_check) < 0 ): raise Exception( - f"VersionError: PaddlePaddle version in [{min_version}, {max_version}] required, but {paddle_version.full_version} installed." + f"VersionError: PaddlePaddle version in [{min_version}, {max_version}] required, but {paddle.version.full_version} installed." ) else: if version_cmp(version_installed, min_version_to_check) < 0: raise Exception( - f"VersionError: PaddlePaddle version {min_version} or higher is required, but {paddle_version.full_version} installed, " + f"VersionError: PaddlePaddle version {min_version} or higher is required, but {paddle.version.full_version} installed, " f"please upgrade your PaddlePaddle to {min_version} or other higher version." ) diff --git a/python/paddle/nn/quant/quantized_linear.py b/python/paddle/nn/quant/quantized_linear.py index 1c2d962f720cf..41ad1839e1f8a 100644 --- a/python/paddle/nn/quant/quantized_linear.py +++ b/python/paddle/nn/quant/quantized_linear.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle import _C_ops, version +import paddle +from paddle import _C_ops from paddle.base.data_feeder import check_dtype from paddle.base.framework import convert_np_dtype_to_dtype_ from paddle.device.cuda import get_device_capability @@ -24,7 +25,7 @@ def _get_arch_info(): # Get SMVersion from device. - cuda_version = version.cuda() + cuda_version = paddle.version.cuda() if cuda_version is not None and cuda_version != 'False': major, minor = get_device_capability() arch = int(major * 10 + minor) diff --git a/python/paddle/tensor/tensor.prototype.pyi b/python/paddle/tensor/tensor.prototype.pyi index 735c8da282545..9b011b602b5e3 100644 --- a/python/paddle/tensor/tensor.prototype.pyi +++ b/python/paddle/tensor/tensor.prototype.pyi @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The `Tensor` template for `tools/gen_tensor_stub.py` generates the stub file `tensor.pyi`. -# Add docstring, attributes, methods and alias with type annotaions for `Tensor` +# The `Tensor` template `tensor.prototype.pyi` for `tools/gen_tensor_stub.py` to generate the stub file `tensor.pyi`. +# Add docstring, attributes, methods and alias with type annotaions for `Tensor` in `tensor.prototype.pyi` # if not conveniently coding in original place (like c++ source file). from typing import Any, overload diff --git a/python/setup.py.in b/python/setup.py.in index 67d23a089aa37..98ccf8c61e41c 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1054,6 +1054,36 @@ if '${WITH_STRIP}' == 'ON': if os.system(command) != 0: raise Exception("strip *.so failed, command: %s" % command) + +def check_build_dependency(): + missing_modules = '''Missing build dependency: {dependency} +Please run 'pip install -r python/requirements.txt' to make sure you have all the dependencies installed. +'''.strip() + + with open('${PADDLE_SOURCE_DIR}' + '/python/requirements.txt') as f: + build_dependencies = ( + f.read().splitlines() + ) # Specify the dependencies to install + + python_dependencies_module = [] + installed_packages = [] + + for dependency in build_dependencies: + python_dependencies_module.append( + re.sub("_|-", '', re.sub(r"==.*|>=.*|<=.*", '', dependency)) + ) + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + + for r in reqs.split(): + installed_packages.append( + re.sub("_|-", '', r.decode().split('==')[0]).lower() + ) + + for dependency in python_dependencies_module: + if dependency.lower() not in installed_packages: + raise RuntimeError(missing_modules.format(dependency=dependency)) + + def install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir): """install cpp distribution and build test target @@ -1095,6 +1125,9 @@ def install_cpp_dist_and_build_test(paddle_install_dir, paddle_lib_test_dir): subprocess.check_call(["cmake", "--build", paddle_lib_test_dir]) +# check build dependency +check_build_dependency() + # install cpp distribution if '${WITH_CPP_DIST}' == 'ON': paddle_install_dir = '${PADDLE_INSTALL_DIR}' @@ -1112,6 +1145,28 @@ package_data['paddle.base'] = package_data.get('paddle.base', []) + [ package_data['paddle.tensor'] = package_data.get('paddle.tensor', []) + ['tensor.pyi'] +def generate_tensor_stub(paddle_binary_dir, paddle_source_dir): + print('-'*2, 'Generate stub file tensor.pyi ... ') + script_path = paddle_source_dir + '/tools/' + sys.path.append(script_path) + import gen_tensor_stub + + gen_tensor_stub.generate_stub_file( + input_file=paddle_source_dir + + '/python/paddle/tensor/tensor.prototype.pyi', + output_file=paddle_binary_dir + '/python/paddle/tensor/tensor.pyi', + ) + + shutil.copy( + paddle_binary_dir + '/python/paddle/tensor/tensor.pyi', + paddle_source_dir + '/python/paddle/tensor/tensor.pyi', + ) + print('-'*2, 'End Generate stub file tensor.pyi ... ') + +# generate stub file `tensor.pyi` +generate_tensor_stub('${PADDLE_BINARY_DIR}', '${PADDLE_SOURCE_DIR}') + + with redirect_stdout(): setup(name='${PACKAGE_NAME}', version='${PADDLE_VERSION}', diff --git a/setup.py b/setup.py index aab6fe0bcfd82..6d9ce542c6a15 100644 --- a/setup.py +++ b/setup.py @@ -1796,6 +1796,25 @@ def submodules_not_exists_or_empty(folder): sys.exit(1) +def generate_tensor_stub(paddle_binary_dir, paddle_source_dir): + print('-' * 2, 'Generate stub file tensor.pyi ... ') + script_path = paddle_source_dir + '/tools/' + sys.path.append(script_path) + import gen_tensor_stub + + gen_tensor_stub.generate_stub_file( + input_file=paddle_source_dir + + '/python/paddle/tensor/tensor.prototype.pyi', + output_file=paddle_binary_dir + '/python/paddle/tensor/tensor.pyi', + ) + + shutil.copy( + paddle_binary_dir + '/python/paddle/tensor/tensor.pyi', + paddle_source_dir + '/python/paddle/tensor/tensor.pyi', + ) + print('-' * 2, 'End Generate stub file tensor.pyi ... ') + + def main(): # Parse the command line and check arguments before we proceed with building steps and setup parse_input_command(filter_args_list) @@ -1875,6 +1894,9 @@ def main(): package_data['paddle.libs'], ) + # generate stub file `tensor.pyi` + generate_tensor_stub(paddle_binary_dir, paddle_source_dir) + setup( name=package_name, version=paddle_version, diff --git a/tools/gen_tensor_stub.py b/tools/gen_tensor_stub.py index 00c7fb0c2e50c..422b3004f5266 100644 --- a/tools/gen_tensor_stub.py +++ b/tools/gen_tensor_stub.py @@ -15,17 +15,18 @@ from __future__ import annotations import argparse +import importlib import inspect import logging import re +import sys +import types from dataclasses import dataclass from functools import cached_property, lru_cache from typing import Any, Callable, Literal from typing_extensions import TypeAlias -import paddle - logging.basicConfig(style="{", format="{message}", level=logging.INFO) logger = logging.getLogger("Generating stub file for paddle.Tensor") logger.setLevel(logging.INFO) @@ -102,7 +103,6 @@ def find_apis(self, api_name: str) -> list[dict[str, tuple[str, int, int]]]: api = [] for mo in pattern.finditer(self._template): _indent = mo.group('indent') - _def_api = mo.group('def_api') _signature = mo.group('signature') _docstring = mo.group('docstring') _ellipsis = mo.group('ellipsis') @@ -110,26 +110,15 @@ def find_apis(self, api_name: str) -> list[dict[str, tuple[str, int, int]]]: _comment = '' if _comment is None else _comment _start_index, _end_index = mo.span() - - _start_indent = _start_index - _end_indent = _start_indent + len(_indent) - - _start_def_api = _end_indent - _end_def_api = _start_def_api + len(_def_api) - - _start_signature = _end_def_api - _end_signature = _start_signature + len(_signature) - - _start_docstring = _end_signature - _end_docstring = _start_docstring + len(_docstring) - - _start_ellipsis = _end_docstring - _end_ellipsis = _start_ellipsis + len(_ellipsis) - + _start_indent, _end_indent = mo.span('indent') + _start_signature, _end_signature = mo.span('signature') + _start_docstring, _end_docstring = mo.span('docstring') + _start_ellipsis, _end_ellipsis = mo.span('ellipsis') _start_comment = _end_ellipsis _end_comment = _start_comment + len(_comment) - assert _end_index == _end_comment + assert _start_index == _start_indent + assert _end_comment == _end_index _api = { 'indent': (_indent, _start_indent, _end_indent), @@ -216,7 +205,10 @@ def add_doc(self, doc: str): self.insert_template(docstring, _end_index, _end_index) def codegen(self) -> str: - return self._template + header = ( + '# This file is auto generated by `tools/gen_tensor_stub.py`.\n\n' + ) + return header + self._template def is_inherited_member(name: str, cls: type) -> bool: @@ -336,7 +328,27 @@ def func_doc_to_method_doc(func_doc: str) -> str: return method_doc +def try_import_paddle() -> types.ModuleType | None: + try: + return importlib.import_module('paddle') + except ModuleNotFoundError: + sys.stderr.write( + '''ERROR: Can NOT import paddle. + We could import paddle without installation, with all libs (.dll or .so) copied into dir `paddle/libs`, + or path already been set for the system. + ''' + ) + + def get_tensor_members(): + paddle = try_import_paddle() + if not paddle: + raise ( + ModuleNotFoundError( + 'Can NOT import paddle from tools/gen_tensor_stub.py.' + ) + ) + tensor_class = paddle.Tensor members: dict[int, Member] = {} @@ -433,7 +445,7 @@ def get_tensor_template(path: str) -> str: return ''.join(f.readlines()) -def main(): +def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( @@ -442,7 +454,6 @@ def main(): type=str, default="python/paddle/tensor/tensor.prototype.pyi", ) - parser.add_argument( "-o", "--output-file", @@ -452,12 +463,16 @@ def main(): args = parser.parse_args() + return args + + +def generate_stub_file(input_file=None, output_file=None): # Get members of Tensor tensor_members = get_tensor_members() logging.debug(f'total members in Tensor: {len(tensor_members)}') # Get tensor template - tensor_template = get_tensor_template(args.input_file) + tensor_template = get_tensor_template(input_file) # Generate the Tensor stub tensor_gen = TensorGen(tensor_template) @@ -473,9 +488,14 @@ def main(): tensor_gen.add_doc(member.doc) # Write to target file - with open(args.output_file, "w", encoding="utf-8") as f: + with open(output_file, "w", encoding="utf-8") as f: f.write(tensor_gen.codegen()) +def main(): + args = parse_args() + generate_stub_file(args.input_file, args.output_file) + + if __name__ == "__main__": main()