Skip to content
This repository has been archived by the owner on Jan 13, 2024. It is now read-only.

Commit

Permalink
update script, add options --schedule
Browse files Browse the repository at this point in the history
  • Loading branch information
@sdpython
sdpython committed Sep 14, 2015
1 parent 8c6a5d4 commit b5a40e3
Show file tree
Hide file tree
Showing 7 changed files with 217 additions and 66 deletions.
102 changes: 102 additions & 0 deletions _unittests/ut_packaged/test_script_install.py
View file
@@ -0,0 +1,102 @@
"""
@brief test log(time=2s)
skip this test for regular run
"""

import sys
import os
import unittest
import re
import pandas

try:
import src
except ImportError:
path = os.path.normpath(
os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"..",
"..")))
if path not in sys.path:
sys.path.append(path)
import src

try:
import pyquickhelper
except ImportError:
path = os.path.normpath(
os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"..",
"..",
"..",
"pyquickhelper",
"src")))
if path not in sys.path:
sys.path.append(path)
if "PYQUICKHELPER" in os.environ and len(os.environ["PYQUICKHELPER"]) > 0:
sys.path.append(os.environ["PYQUICKHELPER"])
import pyquickhelper


from pyquickhelper import fLOG, df2rst
from src.pymyinstall.installhelper import run_cmd


class TestScriptInstall(unittest.TestCase):

def test_script_help(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")

script = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "scripts", "pymy_install.py")
assert os.path.exists(script)
scriptu = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "scripts", "pymy_update.py")
assert os.path.exists(script)

exe = sys.executable

cmd = exe + " " + script + " --help"
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
assert "usage: pymy_install.py" in out

cmd = exe + " " + scriptu + " --help"
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
assert "usage: pymy_update.py" in out

def test_script_schedule(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")

script = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "scripts", "pymy_install.py")
assert os.path.exists(script)
scriptu = os.path.join(os.path.dirname(os.path.abspath(
__file__)), "..", "..", "src", "pymyinstall", "scripts", "pymy_update.py")
assert os.path.exists(script)

exe = sys.executable

cmd = exe + " " + script + " --schedule --set=minimal"
out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
fLOG(out)
fLOG("---")
fLOG(err)
assert "check module: flake8" in out

cmd = exe + " " + scriptu + " --schedule --set=minimal"
out, err = run_cmd(cmd, wait=True)
assert "check module: flake8" in out


if __name__ == "__main__":
unittest.main()
36 changes: 25 additions & 11 deletions src/pymyinstall/installhelper/module_install_version.py
View file
Expand Up @@ -5,6 +5,7 @@
import sys
import re
import pip._vendor.pkg_resources
import warnings

if sys.version_info[0] == 2:
import urllib2 as urllib_request
Expand Down Expand Up @@ -254,52 +255,65 @@ def get_pypi_version(module_name, full_list=False, url="http://pypi.python.org/p
ProtocolError: ProtocolError for pypi.python.org/pypi: 503 No healthy backends
"""

global _get_pypi_version_memoize
key = module_name, full_list, url
if key in _get_pypi_version_memoize:
available = _get_pypi_version_memoize[key]
else:

pypi = xmlrpc_client.ServerProxy(url)

def pypi_package_releases(module_name, b):
nbtry = 0
while nbtry < 2:
try:
available = pypi.package_releases(module_name, True)
return available
except TimeoutError as e:
nbtry += 1
warnings.warn(e)
return None

tried = [module_name]
available = pypi.package_releases(module_name, True)
available = pypi_package_releases(module_name, True)

if available is None or len(available) == 0:
tried.append(module_name.capitalize())
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)

if available is None or len(available) == 0:
tried.append(module_name.replace("-", "_"))
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)

if available is None or len(available) == 0:
tried.append(module_name.replace("_", "-"))
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)

if available is None or len(available) == 0:
tried.append(module_name.lower())
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)

if available is None or len(available) == 0:
ml = module_name.lower()
if ml == "markupsafe":
tried.append("MarkupSafe")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "flask-sqlalchemy":
tried.append("Flask-SQLAlchemy")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "apscheduler":
tried.append("APScheduler")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "datashape":
tried.append("DataShape")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "pycontracts":
tried.append("PyContracts")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "pybrain":
tried.append("PyBrain")
available = pypi.package_releases(tried[-1], True)
available = pypi_package_releases(tried[-1], True)
elif ml == "jsanimation": # github
tried.append("JSAnimation")
available = ["-"]
Expand Down
95 changes: 56 additions & 39 deletions src/pymyinstall/packaged/automate_install.py
View file
Expand Up @@ -80,18 +80,19 @@ def reorder_module_list(list_module):

def update_all(temp_folder=".", fLOG=print, verbose=True,
list_module=None, reorder=True,
skip_module=None):
skip_module=None, schedule_only=False):
"""
update modules in *list_module*
if None, this list will be returned by @see fn ensae_fullset,
the function starts by updating pip.
@param temp_folder temporary folder
@param verbose more display
@param list_module None or of list of str or @see cl ModuleInstall or a name set
@param fLOG logging function
@param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
@param skip_module module to skip (list of str)
@param temp_folder temporary folder
@param verbose more display
@param list_module None or of list of str or @see cl ModuleInstall or a name set
@param fLOG logging function
@param reorder reorder the modules to update first modules with less dependencies (as much as as possible)
@param skip_module module to skip (list of str)
@param schedule_only if True, the function returns the list of modules scheduled to be installed
*list_module* can be a set of modules of a name set.
Name sets can be accesses by function @see fn get_package_set.
Expand All @@ -117,8 +118,8 @@ def update_all(temp_folder=".", fLOG=print, verbose=True,
list_module = ensae_fullset()
elif isinstance(list_module, str # unicode#
):
from .packaged_config import get_name_set
f = get_name_set(list_module)
from .packaged_config import get_package_set
f = get_package_set(list_module)
list_module = f()
else:
list_module = [find_module_install(mod) if isinstance(
Expand All @@ -138,6 +139,7 @@ def update_all(temp_folder=".", fLOG=print, verbose=True,
modules = list_module
again = []
errors = []
schedule = []
for mod in modules:
if verbose:
fLOG("check module: ", mod.name)
Expand All @@ -157,19 +159,25 @@ def update_all(temp_folder=".", fLOG=print, verbose=True,

ver = mod.get_pypi_version()
inst = mod.get_installed_version()
m = " - updating module {0} --- {1} --> {2} (kind={3})" \
.format(mod.name, inst, ver, mod.kind)
fLOG(m)
try:
b = mod.update(temp_folder=temp_folder, log=verbose)
except Exception as e:
b = False
m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4}" \
.format(mod.name, inst, ver, mod.kind, str(e))

schedule.append(mod)
if not schedule_only:
m = " - updating module {0} --- {1} --> {2} (kind={3})" \
.format(mod.name, inst, ver, mod.kind)
fLOG(m)
errors.append((mod, e))
if b:
again.append(m)
try:
b = mod.update(temp_folder=temp_folder, log=verbose)
except Exception as e:
b = False
m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4}" \
.format(mod.name, inst, ver, mod.kind, str(e))
fLOG(m)
errors.append((mod, e))
if b:
again.append(m)

if schedule_only:
return schedule

if verbose:
fLOG("")
Expand All @@ -184,7 +192,8 @@ def update_all(temp_folder=".", fLOG=print, verbose=True,

def install_all(temp_folder=".", fLOG=print, verbose=True,
list_module=None, reorder=True, skip_module=None,
up_pip=True, skip_missing=False, deps=False):
up_pip=True, skip_missing=False, deps=False,
schedule_only=False):
"""
install modules in *list_module*
if None, this list will be returned by @see fn ensae_fullset,
Expand All @@ -199,6 +208,7 @@ def install_all(temp_folder=".", fLOG=print, verbose=True,
@param up_pip upgrade pip (pip must not be in *skip_module*)
@param skip_missing skip the checking of the missing dependencies
@param deps install the dependencies of the installed modules
@param schedule_only if True, the function returns the list of modules scheduled to be installed
*list_module* can be a set of modules of a name set.
Name sets can be accesses by function @see fn get_package_set.
Expand All @@ -219,8 +229,8 @@ def install_all(temp_folder=".", fLOG=print, verbose=True,
list_module = ensae_fullset()
elif isinstance(list_module, str # unicode#
):
from .packaged_config import get_name_set
f = get_name_set(list_module)
from .packaged_config import get_package_set
f = get_package_set(list_module)
list_module = f()
else:
list_module = [find_module_install(mod) if isinstance(
Expand All @@ -240,30 +250,37 @@ def install_all(temp_folder=".", fLOG=print, verbose=True,
again = []
errors = []
installed = []
schedule = []
for mod in modules:
if verbose:
fLOG("check module: ", mod.name)
if not mod.is_installed():
ver = mod.version
m = " - installing module {0} --- --> {1} (kind={2})" \
.format(mod.name, ver, mod.kind)
fLOG(m)
try:
b = mod.install(temp_folder=temp_folder, log=verbose)
except Exception as e:
b = False
m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4}" \
.format(mod.name, '', ver, mod.kind, str(e))
schedule.append(mod)
if not schedule_only:
ver = mod.version
m = " - installing module {0} --- --> {1} (kind={2})" \
.format(mod.name, ver, mod.kind)
fLOG(m)
errors.append((mod, e))
if b:
again.append(m)
installed.append(mod)
try:
b = mod.install(temp_folder=temp_folder, log=verbose)
except Exception as e:
b = False
m = " - failed to update module {0} --- {1} --> {2} (kind={3}) due to {4}" \
.format(mod.name, '', ver, mod.kind, str(e))
fLOG(m)
errors.append((mod, e))
if b:
again.append(m)
installed.append(mod)

if schedule_only:
return schedule

if deps:
fLOG("dependencies")
for mod in installed:
install_module_deps(mod.name, temp_folder=temp_folder, fLOG=fLOG, verbose=verbose, deps=deps):
install_module_deps(mod.name, temp_folder=temp_folder,
fLOG=fLOG, verbose=verbose, deps=deps)

if verbose:
fLOG("")
Expand Down
2 changes: 1 addition & 1 deletion src/pymyinstall/packaged/packaged_config.py
View file
Expand Up @@ -80,7 +80,7 @@ def get_package_set(name):
if name not in _module_set_name:
raise ValueError("unable to find set for {0}\navailable sets:\n{1}".format(
name, "\n".join(sorted(_module_set_name.keys()))))
return _module_set_name
return _module_set_name[name]


def name_sets_dataframe(module_set=None):
Expand Down
2 changes: 1 addition & 1 deletion src/pymyinstall/packaged/packaged_config_C_webscraping.py
View file
Expand Up @@ -3,7 +3,7 @@
@file
@brief Defines a set of modules to try
"""
import sys

from ..installhelper.module_install import ModuleInstall


Expand Down

0 comments on commit b5a40e3

Please sign in to comment.