Fixed neutron-db-manage without neutron/tests installed

With 7c0f189 in tree, we made
neutron-db-manage require neutron/tests/tools.py installed. Some
distributions, like RDO, may split the python package into core package
and the one that contains python code needed for testing only (anything
under neutron/tests/), and hence don't guarantee that all setups have
neutron.tests package available.

This fix moves the import_module_recursively function from
neutron.tests.tools into neutron.common.utils because it has non-testing
use cases. All existing cases where we use the function switched to the
new location. The old symbol still works, though triggers a deprecation
warning, and will be removed in the next cycle.

Change-Id: Ia8d91a1704c894bc1f6cf14e6cdd971fab255b62
Closes-Bug: #1612959

neutron/common/utils.py

@@ -22,9 +22,11 @@
 import decimal
 import errno
 import functools
+import importlib
 import math
 import multiprocessing
 import os
+import os.path
 import random
 import signal
 import socket
@@ -46,6 +48,7 @@
 import six
 from stevedore import driver
 
+import neutron
 from neutron._i18n import _, _LE
 from neutron.db import api as db_api
 
@@ -693,3 +696,33 @@ def extract_exc_details(e):
     if args is _NO_ARGS_MARKER:
         return details
     return details % args
+
+
+def import_modules_recursively(topdir):
+    '''Import and return all modules below the topdir directory.'''
+    modules = []
+    for root, dirs, files in os.walk(topdir):
+        for file_ in files:
+            if file_[-3:] != '.py':
+                continue
+
+            module = file_[:-3]
+            if module == '__init__':
+                continue
+
+            import_base = root.replace('/', '.')
+
+            # NOTE(ihrachys): in Python3, or when we are not located in the
+            # directory containing neutron code, __file__ is absolute, so we
+            # should truncate it to exclude PYTHONPATH prefix
+            prefixlen = len(os.path.dirname(neutron.__file__))
+            import_base = 'neutron' + import_base[prefixlen:]
+
+            module = '.'.join([import_base, module])
+            if module not in sys.modules:
+                importlib.import_module(module)
+            modules.append(module)
+
+        for dir_ in dirs:
+            modules.extend(import_modules_recursively(dir_))
+    return modules

neutron/db/migration/models/head.py

@@ -21,10 +21,9 @@
 
 """
 
-# TODO(ihrachys): move the module under neutron/tests
-
 import os.path
 
+from neutron.common import utils
 from neutron.db import address_scope_db  # noqa
 from neutron.db import agents_db  # noqa
 from neutron.db import agentschedulers_db  # noqa
@@ -63,10 +62,9 @@
 from neutron.services.auto_allocate import models as aa_models  # noqa
 from neutron.services.segments import db  # noqa
 from neutron.services.trunk import models as trunk_models  # noqa
-from neutron.tests import tools
 
 
-tools.import_modules_recursively(os.path.dirname(models.__file__))
+utils.import_modules_recursively(os.path.dirname(models.__file__))
 
 
 def get_metadata():

neutron/tests/tools.py

@@ -14,26 +14,25 @@
 #    under the License.
 
 import copy
-import importlib
 import os
 import platform
 import random
 import string
-import sys
 import time
 import warnings
 
+from debtcollector import moves
 import fixtures
 import mock
 import netaddr
 from neutron_lib import constants
 import six
 import unittest2
 
-import neutron
 from neutron.api.v2 import attributes
 from neutron.common import constants as n_const
 from neutron.common import ipv6_utils
+from neutron.common import utils
 from neutron.db import common_db_mixin
 
 
@@ -132,9 +131,6 @@ def restore_hooks(self):
         common_db_mixin.CommonDbMixin._model_query_hooks = self.original_hooks
 
 
-from neutron.common import utils
-
-
 def setup_mock_calls(mocked_call, expected_calls_and_values):
     """A convenient method to setup a sequence of mock calls.
 
@@ -204,36 +200,6 @@ def __neq__(self, other):
         return not self == other
 
 
-def import_modules_recursively(topdir):
-    '''Import and return all modules below the topdir directory.'''
-    modules = []
-    for root, dirs, files in os.walk(topdir):
-        for file_ in files:
-            if file_[-3:] != '.py':
-                continue
-
-            module = file_[:-3]
-            if module == '__init__':
-                continue
-
-            import_base = root.replace('/', '.')
-
-            # NOTE(ihrachys): in Python3, or when we are not located in the
-            # directory containing neutron code, __file__ is absolute, so we
-            # should truncate it to exclude PYTHONPATH prefix
-            prefixlen = len(os.path.dirname(neutron.__file__))
-            import_base = 'neutron' + import_base[prefixlen:]
-
-            module = '.'.join([import_base, module])
-            if module not in sys.modules:
-                importlib.import_module(module)
-            modules.append(module)
-
-        for dir_ in dirs:
-            modules.extend(import_modules_recursively(dir_))
-    return modules
-
-
 def get_random_string(n=10):
     return ''.join(random.choice(string.ascii_lowercase) for _ in range(n))
 
@@ -333,3 +299,8 @@ def reset_random_seed():
 
 def get_random_ipv6_mode():
     return random.choice(n_const.IPV6_MODES)
+
+
+import_modules_recursively = moves.moved_function(
+    utils.import_modules_recursively, 'import_modules_recursively', __name__,
+    version='Newton', removal_version='Ocata')

neutron/tests/unit/common/test_utils.py

@@ -13,7 +13,9 @@
 #    under the License.
 
 import errno
+import os.path
 import re
+import sys
 
 import eventlet
 import mock
@@ -30,6 +32,7 @@
 from neutron.plugins.common import utils as plugin_utils
 from neutron.tests import base
 from neutron.tests.common import helpers
+from neutron.tests.unit import tests
 
 
 class TestParseMappings(base.BaseTestCase):
@@ -778,3 +781,14 @@ def test_attach_exc_details_with_dict_interpolation(self):
     def test_extract_exc_details_no_details_attached(self):
         self.assertIsInstance(
             utils.extract_exc_details(Exception()), six.text_type)
+
+
+class ImportModulesRecursivelyTestCase(base.BaseTestCase):
+
+    def test_object_modules(self):
+        example_module = 'neutron.tests.unit.tests.example.dir.example_module'
+        sys.modules.pop(example_module, None)
+        modules = utils.import_modules_recursively(
+            os.path.dirname(tests.__file__))
+        self.assertIn(example_module, modules)
+        self.assertIn(example_module, sys.modules)

neutron/tests/unit/objects/test_objects.py

@@ -18,9 +18,9 @@
 from oslo_versionedobjects import base as obj_base
 from oslo_versionedobjects import fixture
 
+from neutron.common import utils
 from neutron import objects
 from neutron.tests import base as test_base
-from neutron.tests import tools
 
 
 # NOTE: The hashes in this list should only be changed if they come with a
@@ -55,7 +55,7 @@ def setUp(self):
         super(TestObjectVersions, self).setUp()
         # NOTE(ihrachys): seed registry with all objects under neutron.objects
         # before validating the hashes
-        tools.import_modules_recursively(os.path.dirname(objects.__file__))
+        utils.import_modules_recursively(os.path.dirname(objects.__file__))
 
     def test_versions(self):
         checker = fixture.ObjectVersionChecker(