Skip files prefixed with _ during plugin discovery (facebookresearch#495

)

hydra/core/plugins.py

@@ -160,6 +160,12 @@ def _scan_all_plugins(
                 path=mdl.__path__, prefix=mdl.__name__ + ".", onerror=lambda x: None
             ):
                 try:
+                    module_name = modname.rsplit(".", 1)[-1]
+                    # If module's name starts with "_", do not load the module.
+                    # But if the module's name starts with a "__", then load the
+                    # module.
+                    if module_name.startswith("_") and not module_name.startswith("__"):
+                        continue
                     import_time = timer()
                     m = importer.find_module(modname)
                     loaded_mod = m.load_module(modname)

news/494.feature

@@ -0,0 +1 @@
+Modules whose name starts with "_" are skipped during plugin discovery

tests/test_plugins/discovery_test_plugin/hydra_plugins/discovery_test/__not_hidden_plugin.py

@@ -0,0 +1,6 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from hydra.plugins.plugin import Plugin
+
+
+class NotHiddenTestPlugin(Plugin):
+    pass

tests/test_plugins/discovery_test_plugin/hydra_plugins/discovery_test/_hidden_plugin.py

@@ -0,0 +1,6 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+from hydra.plugins.plugin import Plugin
+
+
+class HiddenTestPlugin(Plugin):
+    pass

tests/test_plugins/discovery_test_plugin/tests/test_discovery.py

@@ -18,3 +18,12 @@ def test_number_of_imports(tmpdir: Path) -> None:
     with Path(os.environ["TMP_FILE"]) as f:
         txt = str(f.read_text())
         assert txt.split("\n").count("imported") == 1
+
+
+def test_skipped_imports(tmpdir: Path) -> None:
+    # Tests that modules starting with an "_" (but not "__") are skipped
+
+    discovered_plugins = [x.__name__ for x in Plugins.instance().discover(Plugin)]
+    assert "HiddenTestPlugin" not in discovered_plugins
+
+    assert "NotHiddenTestPlugin" in discovered_plugins

website/docs/advanced/plugins.md

@@ -7,8 +7,13 @@ sidebar_label: Hydra plugins
 Hydra can be extended via plugins.
 You can see example plugins [here](https://github.com/facebookresearch/hydra/tree/master/plugins/examples).
 
-## Plugin types
+## Plugin discovery
+During plugin discovery, Hydra looks up for plugins in all submodules of `hydra_plugins`. To do this, Hydra imports sub-modules defined under `hydra_plugins` to look for plugins that are defined in them.
+Since plugins are discovered whenever Hydra starts, any installed plugins that are slow to import will slow down the startup of _ALL_ Hydra applicaitons.
+Plugins with expensive imports can exclude individual files from this by prefixing them with `_` (but not `__`).
+For example, the file `_my_plugin_lib.py` would not be imported and scanned, while `my_plugin_lib.py` would be.
 
+## Plugin types
 ### Sweeper
 A sweeper is responsible for converting command line arguments list into multiple jobs.
 For example, the basic built-in sweeper takes arguments like:
@@ -42,4 +47,4 @@ Many other plugins also implement SearchPathPlugin to add their configuration to
 
 ### ConfigSource
 ConfigSource plugins can be used to allow Hydra to access configuration in non-standard locations when composing the config.
-This can be used to enable to access an in-house private config store, or as a way to access configs from public sources like GitHub or S3.
+This can be used to enable to access an in-house private config store, or as a way to access configs from public sources like GitHub or S3.

website/sidebars.js

@@ -59,6 +59,7 @@ module.exports = {
 
         'Advanced': [
             'advanced/app_packaging',
+            'advanced/plugins',
             'advanced/search_path',
 
         ],