Add plugin tests back (#2207)

.circleci/config.yml

@@ -131,6 +131,9 @@ jobs:
             conda activate hydra
             pip install nox dataclasses --progress-bar off
             nox -s lint test_tools test_core test_jupyter_notebooks -ts
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              SKIP_PLUGINS=hydra_ray_launcher nox -s lint_plugins test_plugins -ts
+            fi
   test_linux:
     parameters:
       py_version:
@@ -147,6 +150,9 @@ jobs:
             export NOX_PYTHON_VERSIONS=<< parameters.py_version >>
             pip install nox dataclasses --progress-bar off
             nox -s lint test_tools test_core test_jupyter_notebooks -ts
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              SKIP_PLUGINS=hydra_ray_launcher nox -s lint_plugins test_plugins -ts
+            fi
   test_win:
     parameters:
       py_version:
@@ -163,6 +169,7 @@ jobs:
             $env:PYTHONIOENCODING="utf_8"
             conda activate hydra
             nox -s lint test_tools test_core test_jupyter_notebooks -ts
+            If ($CIRCLE_PR_NUMBER) {SKIP_PLUGINS=hydra_ray_launcher nox -s lint_plugins test_plugins -ts}
             exit $LASTEXITCODE
   trigger_plugin_pipelines:
     docker:
@@ -194,6 +201,7 @@ jobs:
               export PLUGINS=<< parameters.test_plugin >>
               conda activate hydra
               pip install nox dataclasses --progress-bar off
+              
               nox -s lint_plugins test_plugins -ts
   test_plugin_linux:
     parameters:

noxfile.py

@@ -28,6 +28,7 @@
 # Allow limiting testing to specific plugins
 # The list ['ALL'] indicates all plugins
 PLUGINS = os.environ.get("PLUGINS", "ALL").split(",")
+SKIP_PLUGINS = os.environ.get("SKIP_PLUGINS", "")
 
 SKIP_CORE_TESTS = "0"
 SKIP_CORE_TESTS = os.environ.get("SKIP_CORE_TESTS", SKIP_CORE_TESTS) != "0"
@@ -55,6 +56,7 @@ def get_current_os() -> str:
 print(f"Operating system\t:\t{get_current_os()}")
 print(f"NOX_PYTHON_VERSIONS\t:\t{PYTHON_VERSIONS}")
 print(f"PLUGINS\t\t\t:\t{PLUGINS}")
+print(f"SKIP_PLUGINS\t\t\t:\t{SKIP_PLUGINS}")
 print(f"SKIP_CORE_TESTS\t\t:\t{SKIP_CORE_TESTS}")
 print(f"FIX\t\t\t:\t{FIX}")
 print(f"VERBOSE\t\t\t:\t{VERBOSE}")
@@ -150,7 +152,10 @@ def select_plugins(session, directory: str) -> List[Plugin]:
     ret = []
     skipped = []
     for plugin in plugins:
-        if not (plugin["dir_name"] in PLUGINS or PLUGINS == ["ALL"]):
+        if (
+            not (plugin["dir_name"] in PLUGINS or PLUGINS == ["ALL"])
+            or plugin["dir_name"] in SKIP_PLUGINS
+        ):
             skipped.append(f"Deselecting {plugin['dir_name']}: User request")
             continue