From 9c87bd1fb5d4e1d766632d78efedd938acb8d086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 4 Jan 2018 00:33:57 +0100 Subject: [PATCH] add function create_folder_setup, documentation, API --- _doc/sphinxdoc/source/API/distribution.rst | 4 + _doc/sphinxdoc/source/API/fix.rst | 4 + _doc/sphinxdoc/source/API/index.rst | 9 + _doc/sphinxdoc/source/API/install.rst | 122 +++++++++++ .../blog/2017/2017-11-08_torch_windows.rst | 3 + _doc/sphinxdoc/source/index.rst | 11 +- _unittests/ut_setup/test_setup_folder.py | 69 +++++++ src/pymyinstall/installhelper/__init__.py | 4 +- src/pymyinstall/installhelper/run_cmd.py | 2 +- src/pymyinstall/setuphelper/__init__.py | 2 +- src/pymyinstall/setuphelper/setup_creation.py | 195 +++++++++++++++--- 11 files changed, 390 insertions(+), 35 deletions(-) create mode 100644 _doc/sphinxdoc/source/API/distribution.rst create mode 100644 _doc/sphinxdoc/source/API/fix.rst create mode 100644 _doc/sphinxdoc/source/API/index.rst create mode 100644 _doc/sphinxdoc/source/API/install.rst create mode 100644 _unittests/ut_setup/test_setup_folder.py diff --git a/_doc/sphinxdoc/source/API/distribution.rst b/_doc/sphinxdoc/source/API/distribution.rst new file mode 100644 index 00000000..fcdf6290 --- /dev/null +++ b/_doc/sphinxdoc/source/API/distribution.rst @@ -0,0 +1,4 @@ +Distribution +============ + +.. autosignature:: pymyinstall.win_installer.win_setup_main.win_python_setup diff --git a/_doc/sphinxdoc/source/API/fix.rst b/_doc/sphinxdoc/source/API/fix.rst new file mode 100644 index 00000000..649258f4 --- /dev/null +++ b/_doc/sphinxdoc/source/API/fix.rst @@ -0,0 +1,4 @@ +Fix +=== + +.. autosignature:: pymyinstall.fix.releases.fix_scipy10_for_statsmodels08 diff --git a/_doc/sphinxdoc/source/API/index.rst b/_doc/sphinxdoc/source/API/index.rst new file mode 100644 index 00000000..d2507e54 --- /dev/null +++ b/_doc/sphinxdoc/source/API/index.rst @@ -0,0 +1,9 @@ +API +=== + +.. toctree:: + :maxdepth: 2 + + fix + install + setup diff --git a/_doc/sphinxdoc/source/API/install.rst b/_doc/sphinxdoc/source/API/install.rst new file mode 100644 index 00000000..8d23f8c1 --- /dev/null +++ b/_doc/sphinxdoc/source/API/install.rst @@ -0,0 +1,122 @@ +Installation +============ + +.. contents:: + :local: + +Modules ++++++++ + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.add_shortcut_to_desktop_for_module + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.create_virtual_env + +.. autosignature:: pymyinstall.installhelper.module_install_version.get_module_dependencies + +.. autosignature:: pymyinstall.installhelper.from .status_helper.get_installed_modules + +.. autosignature:: pymyinstall.installhelper.module_install_version.get_module_metadata + +.. autosignature:: pymyinstall.installhelper.module_install_version.get_module_version + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.get_pip_program + +.. autosignature:: pymyinstall.installhelper.module_install_version.get_pypi_version + +.. autosignature:: pymyinstall.installhelper.module_install_version.get_wheel_version + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.has_pip + +.. autosignature:: pymyinstall.installhelper.module_install_version.is_installed + +.. autosignature:: pymyinstall.installhelper.module_install_version.numeric_version + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.run_cmd + +.. autosignature:: pymyinstall.installhelper.install_venv_helper.run_cmd_path + +.. autosignature:: pymyinstall.installhelper.module_dependencies.missing_dependencies + +.. autosignature:: pymyinstall.installhelper.module_install.ModuleInstall + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.run_venv_script + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.update_pip + +.. autosignature:: pymyinstall.installhelper.install_cmd_helper.venv_install + +.. autosignature:: pymyinstall.installhelper.module_install_version.version_consensus + +.. autosignature:: pymyinstall.installhelper.module_install_version.compare_version + +Install ++++++++ + +.. autosignature:: pymyinstall.packaged.packaged_config.classifiers2string + +.. autosignature:: pymyinstall.packaged.automate_install.download_module + +.. autosignature:: pymyinstall.packaged.automate_install.find_module_install + +.. autosignature:: pymyinstall.packaged.packaged_config.get_package_set + +.. autosignature:: pymyinstall.packaged.automate_install.install_module + +.. autosignature:: pymyinstall.packaged.automate_install.install_module_deps + +.. autosignature:: pymyinstall.packaged.automate_install.install_all + +.. autosignature:: pymyinstall.packaged.config_helper.is_64bit + +.. autosignature:: pymyinstall.packaged.packaged_config.name_sets_dataframe + +.. autosignature:: pymyinstall.packaged.automate_install.update_all + +.. autosignature:: pymyinstall.packaged.automate_install.update_module + +Tools ++++++ + +.. autosignature:: pymyinstall.installcustom.install_custom.download_page + +.. autosignature:: pymyinstall.installcustom.install_custom.download_page + +.. autosignature:: pymyinstall.installcustom.install_custom_7z.install_7z + +.. autosignature:: pymyinstall.installcustom.install_custom_chromedriver.install_chromedriver + +.. autosignature:: pymyinstall.installcustom.install_custom_git.install_git + +.. autosignature:: pymyinstall.installcustom.install_custom_graphviz.install_graphviz + +.. autosignature:: pymyinstall.installcustom.install_custom_inkscape.install_inkscape + +.. autosignature:: pymyinstall.installcustom.install_custom_javajdk.install_javajdk + +.. autosignature:: pymyinstall.installcustom.install_custom_jenkins.install_jenkins + +.. autosignature:: pymyinstall.installcustom.install_custom_julia.install_julia + +.. autosignature:: pymyinstall.installcustom.install_custom_miktex.install_miktex + +.. autosignature:: pymyinstall.installcustom.install_custom_mingw.install_mingw + +.. autosignature:: pymyinstall.installcustom.install_custom_operadriver.install_operadriver + +.. autosignature:: pymyinstall.installcustom.install_custom_pandoc.install_pandoc + +.. autosignature:: pymyinstall.installcustom.install_custom_putty.install_putty + +.. autosignature:: pymyinstall.installcustom.install_custom_python.install_python + +.. autosignature:: pymyinstall.installcustom.install_custom_R.install_R + +.. autosignature:: pymyinstall.installcustom.install_custom_scite.install_scite + +.. autosignature:: pymyinstall.installcustom.install_custom_sqlitespy.install_sqlitespy + +.. autosignature:: pymyinstall.installcustom.install_custom_sbt.install_scala_sbt + +.. autosignature:: pymyinstall.installcustom.install_custom_scite.modify_scite_properties + +.. autosignature:: pymyinstall.installcustom.install_custom.where_in_path diff --git a/_doc/sphinxdoc/source/blog/2017/2017-11-08_torch_windows.rst b/_doc/sphinxdoc/source/blog/2017/2017-11-08_torch_windows.rst index 8694dca1..48aa1082 100644 --- a/_doc/sphinxdoc/source/blog/2017/2017-11-08_torch_windows.rst +++ b/_doc/sphinxdoc/source/blog/2017/2017-11-08_torch_windows.rst @@ -8,3 +8,6 @@ You can go there `peterjc123/pytorch `_ or type ``conda install -c peterjc123 pytorch``. + the setup can be created after unzipping the sources + and running example + :ref:`Create a setup from installed packages `. diff --git a/_doc/sphinxdoc/source/index.rst b/_doc/sphinxdoc/source/index.rst index 9e158c8e..50e83791 100644 --- a/_doc/sphinxdoc/source/index.rst +++ b/_doc/sphinxdoc/source/index.rst @@ -114,19 +114,18 @@ Installation ``pip install pymyinstall`` -Quick start ------------ +Documentation +------------- .. toctree:: :maxdepth: 1 + API/index i_ex all_notebooks i_faq name_set_table ensae_full_set_table - completed_todoextlist - issues_todoextlist Galleries --------- @@ -141,8 +140,8 @@ Galleries gynotebooks/index blog/blogindex -Index and License ------------------ +Index +----- .. toctree:: :maxdepth: 1 diff --git a/_unittests/ut_setup/test_setup_folder.py b/_unittests/ut_setup/test_setup_folder.py new file mode 100644 index 00000000..e27071ba --- /dev/null +++ b/_unittests/ut_setup/test_setup_folder.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" +@brief test log(time=25s) +""" + +import sys +import os +import unittest + +try: + import src +except ImportError: + path = os.path.normpath( + os.path.abspath( + os.path.join( + os.path.split(__file__)[0], + "..", + ".."))) + if path not in sys.path: + sys.path.append(path) + import src + +try: + import pyquickhelper as skip_ +except ImportError: + path = os.path.normpath( + os.path.abspath( + os.path.join( + os.path.split(__file__)[0], + "..", + "..", + "..", + "pyquickhelper", + "src"))) + if path not in sys.path: + sys.path.append(path) + if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0: + sys.path.append(os.environ["PYQUICKHELPER"]) + import pyquickhelper as skip_ + + +from src.pymyinstall.setuphelper import create_folder_setup +from pyquickhelper.loghelper import fLOG +from pyquickhelper.pycode import get_temp_folder, ExtTestCase + + +class TestSetupFolder(ExtTestCase): + + def test_setup_folder(self): + fLOG( + __file__, + self._testMethodName, + OutputPrint=__name__ == "__main__") + + temp = get_temp_folder(__file__, "temp_setup_folder", clean=False) + + import pandas + self.assertTrue(pandas is not None) + st = create_folder_setup('pandas', fLOG=fLOG, output_path=temp) + self.assertEqual(len(st), 1) + self.assertExists(st[0]) + exp = os.path.join(temp, 'dist') + found = os.listdir(exp) + self.assertEqual(len(found), 1) + self.assertIn(".whl", found[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/pymyinstall/installhelper/__init__.py b/src/pymyinstall/installhelper/__init__.py index 0dd7ee24..25ccee51 100644 --- a/src/pymyinstall/installhelper/__init__.py +++ b/src/pymyinstall/installhelper/__init__.py @@ -13,8 +13,8 @@ def module_as_table(list_module, as_df=False): """ - returns a list of dictionaries or a dataframe - for a list of modules + Returns a list of dictionaries or a dataframe + for a list of modules. @param list_module list of @see cl ModuleInstall @param as_df as a dataframe or not diff --git a/src/pymyinstall/installhelper/run_cmd.py b/src/pymyinstall/installhelper/run_cmd.py index 8996cfa2..9b226025 100644 --- a/src/pymyinstall/installhelper/run_cmd.py +++ b/src/pymyinstall/installhelper/run_cmd.py @@ -127,7 +127,7 @@ def run_cmd_private(cmd, sin="", shell=True, wait=False, log_error=True, @return content of stdout, stdres (only if wait is True) .. exref:: - :title: Run a program using the command line) + :title: Run a program using the command line :: diff --git a/src/pymyinstall/setuphelper/__init__.py b/src/pymyinstall/setuphelper/__init__.py index 76336112..cd0675f5 100644 --- a/src/pymyinstall/setuphelper/__init__.py +++ b/src/pymyinstall/setuphelper/__init__.py @@ -3,4 +3,4 @@ @brief Shortcut for ``setuphelper`` """ -from .setup_creation import create_empty_folder_setup +from .setup_creation import create_empty_folder_setup, create_folder_setup diff --git a/src/pymyinstall/setuphelper/setup_creation.py b/src/pymyinstall/setuphelper/setup_creation.py index f94455e8..f92931e0 100644 --- a/src/pymyinstall/setuphelper/setup_creation.py +++ b/src/pymyinstall/setuphelper/setup_creation.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ @file @brief Functions to help creating a setup @@ -6,6 +7,7 @@ """ import os import sys +from ..installhelper import run_cmd _setup_py = """ # -*- coding: utf-8 -*- @@ -47,7 +49,7 @@ packages = find_packages('src', exclude='src') package_dir = {k: "src/" + k.replace(".", "/") for k in packages} package_data = { - project_var_name : ["*.pyd", "*.dll" ], + project_var_name : ["*.pyd", "*.dll", "*.so" ], } @@ -105,30 +107,10 @@ def verbose(): **Links:** - * `pypi/__NAME__ <__URL__>`_ - * `GitHub/__NAME__ <__URL__>`_ - * `documentation <__URL__>`_ - * `Travis `_ - * `Blog `_ - - -Getting started ---------------- - -* to be defined - - -Functionalities ---------------- - -* to be defined - - -Versions --------- -* **0.1 - 2016/??/??** - * **new:** first version +* `GitHub/__NAME__ <__URL__>`_ +* `documentation <__URL__>`_ +* `Blog `_ ''' if sys.version_info[0] == 2: @@ -138,7 +120,7 @@ def verbose(): def create_empty_folder_setup(fold, name, author=None, description=None, url=None, durl=None, version="0.1", subversion="0"): """ - create a quick empty shell for a new project + Creates a quick empty shell for a new project. @param fold location @param name name of the project @@ -188,3 +170,166 @@ def adapt(content): writ.append(bat) return writ + + +_setup_py_existing = """ +# -*- coding: utf-8 -*- +import sys +import os +import warnings +from distutils.core import setup +from setuptools import find_packages + +DESCRIPTION = '''__DESCRIPTION__''' + +packages = find_packages('src', exclude='src') +package_dir = {k: 'src/' + k.replace(".", "/") for k in packages} +package_data = { +__PACKDATA__ +} + +setup(name="__FOLDER__", version="__VERSION__", + description=DESCRIPTION, long_description=DESCRIPTION, + packages=packages, package_dir=package_dir, + package_data=package_data +) +""" + + +def create_folder_setup(fold, wheel=True, output_path=None, fLOG=None, version=None, description=None): + """ + Creates a quick setup for an existing or installed projects. + + @param fold folder or module (must be imported first) + @param wheel run the setup to build the wheel + @param output_path copies everything here it not None + @param version to overwrite version + @param description to overwrite description + @return list of created files + + .. exref:: + :title: Create a setup from installed packages + :lid: ex-torch-setup + + This packages :epkg:`pandas` into a wheel from the installed + sources. Location of the sources can be specified too. + + :: + + from pymyinstall.setuphelper import create_folder_setup + create_folder_setup('pandas', fLOG=print, output_path='.') + """ + if fLOG: + fLOG("[create_folder_setup] process '{0}'".format(fold)) + if not os.path.exists(fold): + if fold not in sys.modules: + raise ValueError("Unable to find module '{0}'".format(fold)) + mod = sys.modules[fold] + info = dict(__VERSION__=mod.__version__, + __DESCRIPTION__=mod.__doc__, + __FOLDER__=fold) + fold = os.path.dirname(mod.__file__) + else: + info = dict() + + # Module name + name = os.path.split(fold)[-1] + if fLOG: + fLOG("[create_folder_setup] name='{0}'".format(name)) + fLOG("[create_folder_setup] fold='{0}'".format(fold)) + + if output_path is not None: + # Copies everything. + if fLOG: + fLOG("[create_folder_setup] copy to '{0}'".format(output_path)) + from pyquickhelper.filehelper import synchronize_folder + dest = os.path.join(output_path, 'src', name) + if not os.path.exists(dest): + os.makedirs(dest) + # , filter_copy=lambda name: '__pycache__' not in name) + synchronize_folder(fold, dest) + fold = dest + fold_ = output_path + else: + raise ValueError("output_path must be specified") + + if len(info) == 0 and version is None: + # Import the module. + if name in sys.modules: + del sys.modules[name] + f = os.path.normpath(os.path.abspath(fold)) + f_ = os.path.normpath(os.path.join(fold, '..')) + if fold_ not in sys.path: + sys.path.insert(0, f_) + rem = True + else: + rem = False + + mod = __import__(name) + if rem: + ind = sys.path.index(f_) + del sys.path[ind] + del sys.modules[name] + + info = dict(__VERSION__=mod.__version__, + __DESCRIPTION__=description or mod.__doc__, + __FOLDER__=name) + else: + info = dict(__VERSION__=version, + __DESCRIPTION__=description, + __FOLDER__=name) + + # Package data + from pyquickhelper.filehelper import explore_folder + _, files = explore_folder(fold, fullname=False) + pack = {} + for f in files: + ff = os.path.split(f)[0] + f = os.path.relpath(f, fold) + if '__pycache__' in f: + continue + ext = os.path.splitext(f)[-1] + if ext in {'.py', '.pyc'}: # , '.pyd'}: + continue + f, n = os.path.split(f) + f = f .replace("\\", "/").replace("/", ".") + if f not in pack: + pack[f] = [] + init = os.path.join(ff, "__init__.py") + while not os.path.exists(init): + if fLOG: + fLOG("[create_folder_setup] add '{0}'".format(init)) + with open(init, "w") as fi: + fi.write('# added for additional files') + ff = os.path.split(ff)[0] + init = os.path.join(ff, "__init__.py") + pack[f].append(n) + + if fLOG: + fLOG("[create_folder_setup] add '{0}' in '{1}'".format(n, f)) + + rows = [" '{2}{3}{0}': {1},".format( + k, v, name, '.' if k else '') for k, v in pack.items()] + info['__PACKDATA__'] = "\n".join(rows) + + # Writes setup.py + script = _setup_py_existing + for k, v in info.items(): + if v is not None: + script = script.replace(k, v) + setup = os.path.join(fold_, 'setup.py') + if fLOG: + fLOG("[create_folder_setup] write='{0}'".format(setup)) + with open(setup, "w") as f: + f.write(script) + + if wheel: + cmd = '"{0}" -u setup.py bdist_wheel'.format( + sys.executable.replace("w.exe", ".exe")) + out, err = run_cmd(cmd, wait=True, fLOG=fLOG, change_path=fold_) + if fLOG: + fLOG('[create_folder_setup] OUT --------------\n' + out) + fLOG('[create_folder_setup] ERR --------------\n' + err) + return [setup] + else: + return [setup]