diff --git a/_unittests/ut_cli/test_pymy_deps_cli.py b/_unittests/ut_cli/test_pymy_deps_cli.py
index 4e83817c..e4765a3c 100644
--- a/_unittests/ut_cli/test_pymy_deps_cli.py
+++ b/_unittests/ut_cli/test_pymy_deps_cli.py
@@ -57,7 +57,7 @@ def test_install_deps(self):
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_deps.py"))
- cmd = "{0} {1} {2}".format(
+ cmd = "{0} -u {1} {2}".format(
sys.executable, script, "pandas")
try:
out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
diff --git a/_unittests/ut_cli/test_pymy_install_cli.py b/_unittests/ut_cli/test_pymy_install_cli.py
index 17887467..82e9a9fd 100644
--- a/_unittests/ut_cli/test_pymy_install_cli.py
+++ b/_unittests/ut_cli/test_pymy_install_cli.py
@@ -1,5 +1,5 @@
"""
-@brief test log(time=7s)
+@brief test log(time=23s)
"""
import sys
@@ -58,7 +58,7 @@ def test_install_set_schedule(self):
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_install.py"))
- cmd = "{0} {1} {2}".format(
+ cmd = "{0} -u {1} {2}".format(
sys.executable, script, "--set=pyquickhelper --schedule")
try:
out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
diff --git a/_unittests/ut_cli/test_pymy_install_cli_tool.py b/_unittests/ut_cli/test_pymy_install_cli_tool.py
index 4bc73e1e..47f0bd3d 100644
--- a/_unittests/ut_cli/test_pymy_install_cli_tool.py
+++ b/_unittests/ut_cli/test_pymy_install_cli_tool.py
@@ -1,5 +1,5 @@
"""
-@brief test log(time=1s)
+@brief test log(time=32s)
"""
import sys
@@ -59,7 +59,7 @@ def test_install_tool(self):
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_install.py"))
- cmd = "{0} {1} {2} --force --folder={3}".format(
+ cmd = "{0} -u {1} {2} --force --folder={3}".format(
sys.executable, script, "graphviz --task=tool --source=zip", temp)
out, err = run_cmd(cmd, wait=True)
fLOG("----", cmd)
diff --git a/_unittests/ut_cli/test_pymy_status_cli.py b/_unittests/ut_cli/test_pymy_status_cli.py
new file mode 100644
index 00000000..31bfb49f
--- /dev/null
+++ b/_unittests/ut_cli/test_pymy_status_cli.py
@@ -0,0 +1,84 @@
+"""
+@brief test log(time=7s)
+"""
+
+import sys
+import os
+import unittest
+import warnings
+
+
+try:
+ import src
+except ImportError:
+ path = os.path.normpath(
+ os.path.abspath(
+ os.path.join(
+ os.path.split(__file__)[0],
+ "..",
+ "..")))
+ if path not in sys.path:
+ sys.path.append(path)
+ import src
+
+try:
+ import pyquickhelper as skip_
+except ImportError:
+ path = os.path.normpath(
+ os.path.abspath(
+ os.path.join(
+ os.path.split(__file__)[0],
+ "..",
+ "..",
+ "..",
+ "pyquickhelper",
+ "src")))
+ if path not in sys.path:
+ sys.path.append(path)
+ if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0:
+ sys.path.append(os.environ["PYQUICKHELPER"])
+ import pyquickhelper as skip_
+
+
+from src.pymyinstall.installhelper.install_cmd_helper import run_cmd
+from pyquickhelper.loghelper import fLOG
+from pyquickhelper.pycode import is_travis_or_appveyor, get_temp_folder
+
+
+class TestPyMyStatusCli(unittest.TestCase):
+
+ def test_status(self):
+ fLOG(
+ __file__,
+ self._testMethodName,
+ OutputPrint=__name__ == "__main__")
+
+ if is_travis_or_appveyor() == "travis":
+ warnings.warn("run_cmd no end on travis")
+ return
+ temp = get_temp_folder(__file__, "temp_status")
+ outfile = os.path.join(temp, "modules.xlsx")
+ this = os.path.abspath(os.path.dirname(__file__))
+ script = os.path.normpath(os.path.join(
+ this, "..", "..", "src", "pymyinstall", "cli", "pymy_status.py"))
+ cmd = "{0} -u {1} {2}".format(
+ sys.executable, script, "numpy --out={0}".format(outfile))
+ fLOG(cmd)
+ out, err = run_cmd(cmd, wait=True)
+ if len(out) == 0:
+ if is_travis_or_appveyor() == "appveyor":
+ warnings.warn(
+ "CLI ISSUE cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
+ else:
+ raise Exception(
+ "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
+ if len(err) > 0:
+ raise Exception(
+ "cmd:\n{0}\nOUT:\n{1}\nERR\n{2}".format(cmd, out, err))
+ if not os.path.exists(outfile):
+ raise Exception(outfile)
+ fLOG(out)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/_unittests/ut_cli/test_pymy_update_cli.py b/_unittests/ut_cli/test_pymy_update_cli.py
index 8b1071d2..05fe9508 100644
--- a/_unittests/ut_cli/test_pymy_update_cli.py
+++ b/_unittests/ut_cli/test_pymy_update_cli.py
@@ -1,5 +1,5 @@
"""
-@brief test log(time=1s)
+@brief test log(time=63s)
"""
import sys
@@ -59,7 +59,7 @@ def test_update_set_schedule(self):
this = os.path.abspath(os.path.dirname(__file__))
script = os.path.normpath(os.path.join(
this, "..", "..", "src", "pymyinstall", "cli", "pymy_update.py"))
- cmd = "{0} {1} {2}".format(
+ cmd = "{0} -u {1} {2}".format(
sys.executable, script, "--set=pyquickhelper --schedule")
out, err = run_cmd(cmd, wait=True)
if len(out) == 0:
diff --git a/_unittests/ut_cli/test_script_install_cli.py b/_unittests/ut_cli/test_script_install_cli.py
index d183fd34..38557e2b 100644
--- a/_unittests/ut_cli/test_script_install_cli.py
+++ b/_unittests/ut_cli/test_script_install_cli.py
@@ -1,5 +1,5 @@
"""
-@brief test log(time=2s)
+@brief test log(time=20s)
skip this test for regular run
"""
diff --git a/_unittests/ut_install/test_status_helper.py b/_unittests/ut_install/test_status_helper.py
new file mode 100644
index 00000000..1f4db47a
--- /dev/null
+++ b/_unittests/ut_install/test_status_helper.py
@@ -0,0 +1,70 @@
+"""
+@brief test log(time=20s)
+"""
+
+import sys
+import os
+import unittest
+
+try:
+ import src
+except ImportError:
+ path = os.path.normpath(
+ os.path.abspath(
+ os.path.join(
+ os.path.split(__file__)[0],
+ "..",
+ "..")))
+ if path not in sys.path:
+ sys.path.append(path)
+ import src
+
+try:
+ import pyquickhelper as skip_
+except ImportError:
+ path = os.path.normpath(
+ os.path.abspath(
+ os.path.join(
+ os.path.split(__file__)[0],
+ "..",
+ "..",
+ "..",
+ "pyquickhelper",
+ "src")))
+ if path not in sys.path:
+ sys.path.append(path)
+ if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0:
+ sys.path.append(os.environ["PYQUICKHELPER"])
+ import pyquickhelper as skip_
+
+
+from src.pymyinstall.installhelper import get_installed_modules
+from pyquickhelper.loghelper import fLOG
+
+
+class TestStatusHelper(unittest.TestCase):
+
+ def test_status_helper(self):
+ fLOG(
+ __file__,
+ self._testMethodName,
+ OutputPrint=__name__ == "__main__")
+
+ res = get_installed_modules(
+ fLOG=fLOG, stop=10, pypi=True, short_list=["numpy"])
+ for f in res:
+ fLOG(f)
+ self.assertEqual(len(res), 1)
+
+ res = get_installed_modules(
+ fLOG=fLOG, stop=10, pypi=True, short_list=["dataspyre"])
+ fLOG(res)
+
+ res = get_installed_modules(fLOG=fLOG, stop=10, pypi=True)
+ for f in res:
+ fLOG(f)
+ self.assertEqual(len(res), 10)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/setup.py b/setup.py
index fda8ca19..91d66d80 100644
--- a/setup.py
+++ b/setup.py
@@ -213,12 +213,14 @@ def write_version():
'pymy_update3 = pymyinstall.cli.pymy_update:main',
'pymy_install3 = pymyinstall.cli.pymy_install:main',
'pymy_deps3 = pymyinstall.cli.pymy_deps:main',
+ 'pymy_status3 = pymyinstall.cli.pymy_status:main',
]}
if sys.platform.startswith("win"):
entry_points['console_scripts'].extend([
'pymy_update = pymyinstall.cli.pymy_update:main',
'pymy_install = pymyinstall.cli.pymy_install:main',
'pymy_deps = pymyinstall.cli.pymy_deps:main',
+ 'pymy_status = pymyinstall.cli.pymy_status:main',
])
else:
entry_points = {
@@ -226,6 +228,7 @@ def write_version():
'pymy_update = pymyinstall.cli.pymy_update:main',
'pymy_install = pymyinstall.cli.pymy_install:main',
'pymy_deps = pymyinstall.cli.pymy_deps:main',
+ 'pymy_status = pymyinstall.cli.pymy_status:main',
]}
setup(
diff --git a/src/pymyinstall/cli/pymy_deps.py b/src/pymyinstall/cli/pymy_deps.py
index 2794285f..6b517f0b 100644
--- a/src/pymyinstall/cli/pymy_deps.py
+++ b/src/pymyinstall/cli/pymy_deps.py
@@ -22,7 +22,7 @@
def get_parser():
"""
- defines the way to parse the magic command ``%head``
+ defines the way to parse the script ``pymy_deps``
"""
parser = argparse.ArgumentParser(
description='look dependencies for modules')
diff --git a/src/pymyinstall/cli/pymy_install.py b/src/pymyinstall/cli/pymy_install.py
index 60ad8926..167402e2 100644
--- a/src/pymyinstall/cli/pymy_install.py
+++ b/src/pymyinstall/cli/pymy_install.py
@@ -12,7 +12,7 @@
def get_parser():
"""
- defines the way to parse the magic command ``%head``
+ defines the way to parse the script ``pymy_install``
"""
typstr = str # unicode#
parser = argparse.ArgumentParser(
diff --git a/src/pymyinstall/cli/pymy_status.py b/src/pymyinstall/cli/pymy_status.py
new file mode 100644
index 00000000..15c86b70
--- /dev/null
+++ b/src/pymyinstall/cli/pymy_status.py
@@ -0,0 +1,104 @@
+#!python
+"""
+script which goes through all installed modules
+"""
+from __future__ import print_function
+import sys
+import os
+import argparse
+
+
+def get_parser():
+ """
+ defines the way to parse the script ``pymy_status``
+ """
+ typstr = str # unicode#
+ parser = argparse.ArgumentParser(
+ description='information about installed modules')
+ parser.add_argument(
+ '-o',
+ '--out',
+ default="python_module.xlsx",
+ type=typstr,
+ help='output the results into a file (required pandas)')
+ parser.add_argument(
+ '-p',
+ '--pypi',
+ default=True,
+ type=bool,
+ help='checks the version on PyPi')
+ parser.add_argument(
+ 'module',
+ nargs="*",
+ default="all",
+ help='update only the list of modules included in this list or all modules if not specified or equal to all')
+ return parser
+
+
+def do_main(list_module=None, outfile="python_module.xlsx", pypi=True):
+ """
+ calls function @see fn update_all but is meant to be added to scripts folder
+
+ @param list_module list of modules to update or None for all
+ @param outfile output the results into a flat file or an excel file (required pandas)
+ @param pypi check version on PyPi
+ """
+ try:
+ from pymyinstall import is_travis_or_appveyor
+ except ImportError:
+ def is_travis_or_appveyor():
+ import sys
+ if "travis" in sys.executable:
+ return "travis"
+ import os
+ if os.environ.get("USERNAME", os.environ.get("USER", None)) == "appveyor":
+ return "appveyor"
+ return None
+
+ try:
+ from pymyinstall.installhelper import get_installed_modules
+ except ImportError:
+ pfolder = os.path.normpath(os.path.join(
+ os.path.abspath(os.path.dirname(__file__)), "..", ".."))
+ sys.path.append(pfolder)
+ from pymyinstall.installhelper import get_installed_modules
+ if len(list_module) == 0:
+ list_module = None
+ if outfile:
+ import pandas
+ res = get_installed_modules(short_list=list_module, fLOG=print, pypi=True)
+ if outfile:
+ df = pandas.DataFrame(res)
+ cols = list(df.columns)
+ cols.sort()
+ df = df[cols]
+ if ".xls" in os.path.split(outfile)[-1]:
+ print("Saving into:", outfile)
+ df.to_excel(outfile, index=False)
+ else:
+ print("Saving into (sep=tab, encoding=utf-8):", outfile)
+ df.to_csv(outfile, seo="\t", encoding="utf-8", index=False)
+
+
+def main():
+ """
+ calls function @see fn update_all but is meant to be added to scripts folder,
+ parse command line arguments
+ """
+ parser = get_parser()
+ try:
+ res = parser.parse_args()
+ except SystemExit:
+ print(parser.format_usage())
+ res = None
+
+ if res is not None:
+ list_module = None if res.module in [
+ "all", "", "-", None, []] else res.module
+ if list_module is None and res.set is not None and len(res.set) > 0 and res.set != "-":
+ list_module = res.set
+ do_main(list_module=list_module, outfile=res.out, pypi=res.pypi)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/pymyinstall/cli/pymy_update.py b/src/pymyinstall/cli/pymy_update.py
index 099fabbc..57b91529 100644
--- a/src/pymyinstall/cli/pymy_update.py
+++ b/src/pymyinstall/cli/pymy_update.py
@@ -12,7 +12,7 @@
def get_parser():
"""
- defines the way to parse the magic command ``%head``
+ defines the way to parse the script ``pymy_update``
"""
typstr = str # unicode#
parser = argparse.ArgumentParser(
diff --git a/src/pymyinstall/installhelper/__init__.py b/src/pymyinstall/installhelper/__init__.py
index dff67cfd..2bc9f150 100644
--- a/src/pymyinstall/installhelper/__init__.py
+++ b/src/pymyinstall/installhelper/__init__.py
@@ -10,6 +10,7 @@
from .module_dependencies import missing_dependencies
from .module_install_version import get_module_dependencies, get_module_version, get_pypi_version, get_module_metadata
from .module_install_version import version_consensus, numeric_version, compare_version, is_installed, get_wheel_version
+from .status_helper import get_installed_modules
def module_as_table(list_module, as_df=False):
diff --git a/src/pymyinstall/installhelper/module_install_version.py b/src/pymyinstall/installhelper/module_install_version.py
index 0531ade7..c707a44d 100644
--- a/src/pymyinstall/installhelper/module_install_version.py
+++ b/src/pymyinstall/installhelper/module_install_version.py
@@ -27,12 +27,8 @@
"NLopt"}
-def call_get_installed_distributions(local_only=True,
- skip=None,
- include_editables=True,
- editables_only=False,
- user_only=False,
- use_cmd=False):
+def call_get_installed_distributions(local_only=True, skip=None, include_editables=True,
+ editables_only=False, user_only=False, use_cmd=False):
"""
Direct call to function *get_installed_distributions* from
`pip `_
@@ -58,8 +54,7 @@ def call_get_installed_distributions(local_only=True,
from pip.compat import stdlib_pkgs
skip = stdlib_pkgs
from pip.utils import get_installed_distributions
- return get_installed_distributions(local_only=local_only,
- skip=skip,
+ return get_installed_distributions(local_only=local_only, skip=skip,
include_editables=include_editables,
editables_only=editables_only,
user_only=user_only)
@@ -215,14 +210,15 @@ def helper(module_name, full_list=False, url="https://pypi.python.org/pypi"):
_get_pypi_version_memoize = {}
-def get_pypi_version(module_name, full_list=False, url="https://pypi.python.org/pypi"):
+def get_pypi_version(module_name, full_list=False, url="https://pypi.python.org/pypi", skip_betas=True):
"""
returns the version of a package on pypi,
we skip alpha, beta or dev version
@param module_name module name
- @param url pipy server
+ @param url pypi server
@param full_list results as a list or return the last stable version
+ @param skip_betas skip the intermediate functions
@return version (str or list)
See also `installing_python_packages_programatically.py `_,
@@ -348,20 +344,25 @@ def _inside_loop_(pypi, module_name, tried):
if available is None or len(available) == 0:
raise MissingPackageOnPyPiException("tried:\n" + "\n".join(tried))
- if full_list:
- _get_pypi_version_memoize[key] = available
- return available
-
- for a in available:
+ def filter_betas(a):
spl = a.split(".")
if len(spl) in (2, 3):
last = spl[-1]
- if "a" not in last and "b" not in last and "dev" not in last:
- _get_pypi_version_memoize[key] = available
- return a
+ if skip_betas and "a" not in last and "b" not in last and "dev" not in last:
+ return True
else:
+ # we don't really know here, so we assume it is not
+ return True
+ return False
+
+ if full_list:
+ _get_pypi_version_memoize[key] = list(
+ filter(filter_betas, available))
+ return available
+
+ for a in available:
+ if filter_betas(a):
_get_pypi_version_memoize[key] = available
- return a
raise MissingVersionOnPyPiException(
"{0}\nversion:\n{1}".format(module_name, "\n".join(available)))
diff --git a/src/pymyinstall/installhelper/status_helper.py b/src/pymyinstall/installhelper/status_helper.py
new file mode 100644
index 00000000..d20028f5
--- /dev/null
+++ b/src/pymyinstall/installhelper/status_helper.py
@@ -0,0 +1,100 @@
+"""
+@file
+@brief Functions to get a status on the distribution
+
+.. versionadded:: 1.1
+"""
+import os
+import pip
+import time
+from .module_install_version import get_pypi_version
+from ..packaged import all_set
+from ..packaged.packaged_config import classifiers2string
+
+
+def get_installed_modules(pypi=False, skip_betas=False, fLOG=None, stop=-1, short_list=None):
+ """
+ Returns all modules recored in this modules.
+ Adds installed modules.
+
+ @param pypi add pypi version
+ @param skip_betas skip the intermediate functions
+ @param stop stop after *stop* (or -1 for all)
+ @param short_list short list of modules or None for all
+ @param fLOG logging function
+ @return list of dictionaries
+ """
+ mod = all_set()
+ mod.sort()
+ rows = [_.as_dict(rst_link=True) for _ in mod]
+ for row in rows:
+ row["classifier"] = classifiers2string(row["classifier"])
+ keys = {mod["name"].lower(): mod for mod in rows}
+ keys.update({mod["mname"].lower(): mod for mod in rows if mod["mname"]})
+
+ all_installed = []
+ dists = pip.get_installed_distributions()
+ if short_list:
+ dists = [_ for _ in dists if _.project_name in short_list]
+ pack = map(lambda d: (d.project_name.lower(), d), dists)
+ for i, (lname, dist) in enumerate(sorted(pack)):
+ if stop >= 0 and i >= stop:
+ break
+ res = {}
+ key = dist.project_name.lower()
+ if key in keys:
+ res.update(keys[key])
+
+ names = [dist.key, dist.project_name, dist.key.replace("-", "_"),
+ res.get("mname", None), res.get("name", None)]
+ date = ""
+ location = None
+ for name in names:
+ if name is None:
+ continue
+ ini = os.path.join(dist.location, name, "__init__.py")
+ if os.path.exists(ini):
+ date = time.ctime(os.path.getctime(ini))
+ location = os.path.dirname(ini)
+ break
+ ini = os.path.join(dist.location, name + ".py")
+ if os.path.exists(ini):
+ date = time.ctime(os.path.getctime(ini))
+ location = ini
+ break
+ res["installed"] = dist.project_name
+ if ini is not None:
+ res["location"] = location
+ res["installed_date"] = date
+ res["installed_version"] = dist.version
+
+ # update?
+ if pypi:
+ available = get_pypi_version(
+ dist.project_name, skip_betas=skip_betas, full_list=True)
+
+ if not available:
+ msg = '-'
+ elif available[0] != dist.version:
+ msg = '!='
+ else:
+ msg = "=="
+ res["available"] = available
+ res["pypi"] = available[0]
+ else:
+ msg = ''
+ available = None
+
+ res["diff_pypi"] = msg
+
+ if fLOG:
+ pkg_info = '{dist.project_name:30} {dist.version:20}'.format(
+ dist=dist)
+ key = key if dist.key != dist.project_name else ""
+ mes = '{pkg_info} {msg:3} - {date:20} --- {available} - {location}'
+ mes = mes.format(pkg_info=pkg_info, msg=msg, date=date, dist=dist,
+ available=available, location=res.get('location', ''))
+ fLOG("{0}/{1} - {2}".format(i, len(dists), mes))
+
+ all_installed.append(res)
+ return all_installed