Merge 9ff742f into d7f39cb

travis/getaddons.py

@@ -34,53 +34,59 @@ def is_module(path):
         return False
 
 
-def is_installable_module(path):
-    """return False if the path doesn't contain an installable odoo module,
-    and the full path to the module manifest otherwise"""
-    manifest_path = is_module(path)
-    if manifest_path:
-        manifest = ast.literal_eval(open(manifest_path).read())
-        if manifest.get('installable', True):
-            return manifest_path
-    return False
-
-
 def get_modules(path):
+    """ Return modules of path repo (used in test_server.py)"""
+    return list(get_modules_info(path).keys())
 
+
+def get_modules_info(path):
+    """ Return a digest of each installable module's manifest """
     # Avoid empty basename when path ends with slash
     if not os.path.basename(path):
         path = os.path.dirname(path)
 
-    res = []
+    modules = {}
     if os.path.isdir(path):
-        res = [x for x in os.listdir(path)
-               if is_installable_module(os.path.join(path, x))]
-    return res
+        for module in os.listdir(path):
+            manifest_path = is_module(os.path.join(path, module))
+            if manifest_path:
+                manifest = ast.literal_eval(open(manifest_path).read())
+                if manifest.get('installable', True):
+                    modules[module] = {
+                        'application': manifest.get('application'),
+                        'depends': manifest.get('depends'),
+                        'auto_install': manifest.get('auto_install'),
+                    }
+    return modules
 
 
 def is_addons(path):
     res = get_modules(path) != []
     return res
 
 
-def get_addons(path):
-    if not os.path.exists(path):
+def get_addons(path, depth=1):
+    """Return repositories in path. Can search in inner folders as depth."""
+    if not os.path.exists(path) or depth < 0:
         return []
+    res = []
     if is_addons(path):
-        res = [path]
+        res.append(path)
     else:
-        res = [os.path.join(path, x)
-               for x in sorted(os.listdir(path))
-               if is_addons(os.path.join(path, x))]
+        new_paths = [os.path.join(path, x)
+                     for x in os.listdir(path)
+                     if os.path.isdir(os.path.join(path, x))]
+        for new_path in new_paths:
+            res.extend(get_addons(new_path, depth-1))
     return res
 
 
 def get_modules_changed(path, ref='HEAD'):
-    '''Get modules changed from git diff-index {ref}
+    """Get modules changed from git diff-index {ref}
     :param path: String path of git repo
     :param ref: branch or remote/branch or sha to compare
     :return: List of paths of modules changed
-    '''
+    """
     git_run_obj = GitRun(os.path.join(path, '.git'))
     if ref != 'HEAD':
         fetch_ref = ref
@@ -102,6 +108,61 @@ def get_modules_changed(path, ref='HEAD'):
     return modules_changed_path
 
 
+def get_dependencies(modules, module_name):
+    """Return a set of all the dependencies in deep of the module_name.
+    The module_name is included in the result."""
+    result = set()
+    for dependency in modules.get(module_name, {}).get('depends', []):
+        result |= get_dependencies(modules, dependency)
+    return result | set([module_name])
+
+
+def get_dependents(modules, module_name):
+    """Return a set of all the modules that are dependent of the module_name.
+    The module_name is included in the result."""
+    result = set()
+    for dependent in modules.keys():
+        if module_name in modules.get(dependent, {}).get('depends', []):
+            result |= get_dependents(modules, dependent)
+    return result | set([module_name])
+
+
+def add_auto_install(modules, to_install):
+    """ Append automatically installed glue modules to to_install if their
+    dependencies are already present. to_install is a set. """
+    found = True
+    while found:
+        found = False
+        for module, module_data in modules.items():
+            if (module_data.get('auto_install') and
+                    module not in to_install and
+                    all(dependency in to_install
+                        for dependency in module_data.get('depends', []))):
+                found = True
+                to_install.add(module)
+    return to_install
+
+
+def get_applications_with_dependencies(modules):
+    """ Return all modules marked as application with their dependencies.
+    For our purposes, l10n modules cannot be an application. """
+    result = set()
+    for module in modules.keys():
+        if modules[module]['application'] and not module.startswith('l10n_'):
+            result |= get_dependencies(modules, module)
+    return add_auto_install(modules, result)
+
+
+def get_localizations_with_dependents(modules):
+    """ Return all localization modules with the modules that depend on them
+    """
+    result = set()
+    for module in modules.keys():
+        if module.startswith('l10n_'):
+            result |= get_dependents(modules, module)
+    return result
+
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv
@@ -111,23 +172,53 @@ def main(argv=None):
         return 1
 
     list_modules = False
+    application = None
+    localization = None
     exclude_modules = []
 
     while params and params[0].startswith('-'):
         param = params.pop(0)
         if param == '-m':
             list_modules = True
-        if param == '-e':
+        elif param == '-e':
             exclude_modules = [x for x in params.pop(0).split(',')]
-
-    func = get_modules if list_modules else get_addons
-    lists = [func(x) for x in params]
-    res = [x for l in lists for x in l]  # flatten list of lists
+        elif param == '--only-applications':
+            application = True
+        elif param == '--exclude-applications':
+            application = False
+        elif param == '--only-localization':
+            localization = True
+        elif param == '--exclude-localization':
+            localization = False
+        elif param.startswith('-'):
+            raise Exception('Unknown parameter: %s' % param)
+
+    if list_modules:
+        modules = {}
+        for path in params:
+            modules.update(get_modules_info(path))
+        res = set(modules.keys())
+        applications, localizations = set(), set()
+        if application is True or application is False:
+            applications = get_applications_with_dependencies(modules)
+            if not application:
+                res -= applications
+                applications = set()
+        if localization is True or localization is False:
+            localizations = get_localizations_with_dependents(modules)
+            if not localization:
+                res -= localizations
+                localizations = set()
+        if application or localization:
+            res = applications | localizations
+        res = list(res)
+    else:
+        lists = [get_addons(path) for path in params]
+        res = [x for l in lists for x in l]  # flatten list of lists
     if exclude_modules:
         res = [x for x in res if x not in exclude_modules]
-    result = ','.join(res)
-    print (result)
-    return result
+    print(','.join(res))
+    return res
 
 
 if __name__ == "__main__":

travis/run_pylint.py

@@ -3,16 +3,16 @@
 
 from __future__ import print_function
 
+import ast
 import os
 import re
 import sys
 
 import click
 import pylint.lint
 
-import getaddons
 import travis_helpers
-from getaddons import get_modules_changed
+from getaddons import get_modules_changed, is_module
 from git_run import GitRun
 
 try:
@@ -218,27 +218,39 @@ def get_count_fails(linter_stats, msgs_no_count=None):
         if msg not in msgs_no_count])
 
 
-def get_subpaths(paths):
+def is_installable_module(path):
+    """return False if the path doesn't contain an installable odoo module,
+    and the full path to the module manifest otherwise"""
+    manifest_path = is_module(path)
+    if manifest_path:
+        manifest = ast.literal_eval(open(manifest_path).read())
+        if manifest.get('installable', True):
+            return manifest_path
+    return False
+
+
+def get_subpaths(paths, depth=1):
     """Get list of subdirectories
     if `__init__.py` file not exists in root path then
     get subdirectories.
     Why? More info here:
         https://www.mail-archive.com/code-quality@python.org/msg00294.html
     :param paths: List of paths
+    :param depth: How many folders can be opened in deep to find a module.
     :return: Return list of paths with subdirectories.
     """
     subpaths = []
     for path in paths:
+        if depth < 0:
+            continue
         if not os.path.isfile(os.path.join(path, '__init__.py')):
-            subpaths.extend(
-                [os.path.join(path, item)
-                 for item in os.listdir(path)
-                 if os.path.isfile(os.path.join(path, item, '__init__.py')) and
-                 (not getaddons.is_module(os.path.join(path, item)) or
-                  getaddons.is_installable_module(os.path.join(path, item)))])
+            new_subpaths = [os.path.join(path, item)
+                            for item in os.listdir(path)
+                            if os.path.isdir(os.path.join(path, item))]
+            if new_subpaths:
+                subpaths.extend(get_subpaths(new_subpaths, depth-1))
         else:
-            if not getaddons.is_module(path) or \
-                    getaddons.is_installable_module(path):
+            if is_installable_module(path):
                 subpaths.append(path)
     return subpaths
 

travis/self_tests

@@ -97,22 +97,38 @@ class MainTest(unittest.TestCase):
     def test_get_addons(self):
         self.assertEquals(getaddons.main(), 1)
         self.assertEquals(
-            getaddons.main(["getaddons.py", self.repo_dir]), self.repo_dir)
-        self.assertIsNotNone(
-            getaddons.main(["getaddons.py", "-m", self.repo_dir]))
-        self.assertIsNotNone(
-            getaddons.main(
+            getaddons.main(["getaddons.py", self.repo_dir]), [self.repo_dir])
+        self.assertEquals(
+            len(getaddons.main(["getaddons.py", "-m", self.repo_dir])), 6)
+        self.assertEquals(
+            len(getaddons.main(
                 ["getaddons.py", "-m", self.repo_dir + "/" if
-                 self.repo_dir[-1] == '/' else self.repo_dir]))
+                 self.repo_dir[-1] == '/' else self.repo_dir])), 6)
+        self.assertEquals(
+            len(getaddons.main(
+                ["getaddons.py", "-m", "--only-applications",
+                 self.repo_dir])), 0)
+        self.assertEquals(
+            len(getaddons.main(
+                ["getaddons.py", "-m", "--only-localization",
+                 self.repo_dir])), 0)
+        self.assertEquals(
+            len(getaddons.main(
+                ["getaddons.py", "-m", "--exclude-applications",
+                 "--exclude-localization", self.repo_dir])), 6)
 
     @unittest.skipIf(os.environ.get("EXCLUDE", False) is False, "Set EXCLUDE")
     def test_get_addons_exclude(self):
-        self.assertIsNotNone(
-            getaddons.main(
-                ["getaddons.py", "-m", self.repo_dir, "-e", self.exclude]))
-        self.assertIsNotNone(
+        self.assertEquals(
+            len(getaddons.main(
+                ["getaddons.py", "-m", "-e", self.exclude, self.repo_dir])), 3)
+        self.assertEquals(
+            len(getaddons.main(
+                ["getaddons.py", "-m", self.repo_dir, "-e", self.exclude])), 6)
+        self.assertEquals(
             getaddons.main(
-                ["getaddons.py", "-e", self.exclude, self.repo_dir]))
+                ["getaddons.py", "-e", self.exclude, self.repo_dir]),
+            [self.repo_dir])
 
     def test_addons_path_order(self):
         addon_paths_alfanumerical_order_is = getaddons.get_addons(

travis/test_server.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import ast
 import re
 import os
 import shutil
 import subprocess
 import sys
 from six import string_types
-from getaddons import get_addons, get_modules, is_installable_module
+from getaddons import (
+    get_addons, get_modules, get_modules_info, get_dependencies)
 from travis_helpers import success_msg, fail_msg
 from configparser import ConfigParser
 
@@ -176,16 +176,13 @@ def get_test_dependencies(addons_path, addons_list):
     if not addons_list:
         return ['base']
     else:
+        modules = {}
         for path in addons_path.split(','):
-            manif_path = is_installable_module(
-                os.path.join(path, addons_list[0]))
-            if not manif_path:
-                continue
-            manif = ast.literal_eval(open(manif_path).read())
-            return list(
-                set(manif.get('depends', [])) |
-                set(get_test_dependencies(addons_path, addons_list[1:])) -
-                set(addons_list))
+            modules.update(get_modules_info(path))
+        dependencies = set()
+        for module in addons_list:
+            dependencies |= get_dependencies(modules, module)
+        return list(dependencies - set(addons_list))
 
 
 def cmd_strip_secret(cmd):
@@ -360,8 +357,8 @@ def main(argv=None):
     preinstall_modules = get_test_dependencies(addons_path,
                                                tested_addons_list)
 
-    preinstall_modules = list(set(preinstall_modules or []) - set(get_modules(
-        os.environ.get('TRAVIS_BUILD_DIR')) or [])) or ['base']
+    preinstall_modules = list(set(preinstall_modules) - set(get_modules(
+        os.environ.get('TRAVIS_BUILD_DIR')))) or ['base']
     print("Modules to preinstall: %s" % preinstall_modules)
     setup_server(dbtemplate, odoo_unittest, tested_addons, server_path,
                  script_name, addons_path, install_options, preinstall_modules,