Bryanv/test cleanup (#8091)

* clean up meta.yaml and remove pytest-selenium

* fresh start

* rename quality mark to codebase, clean up codebase tests

* update docs

* docs fix

conda.recipe/meta.yaml

@@ -19,19 +19,19 @@ extra:
 
 requirements:
   build:
-    - python
-    - setuptools
-    - nodejs >=8.8,<9.0
     - certifi
-    - pycurl
+    - jinja2
+    - nodejs >=8.8,<9.0
+    - numpy
     - packaging
-    - six
+    - pycurl
+    - python
     - python-dateutil
-    - numpy
-    - yaml
     - pyyaml
-    - jinja2
+    - setuptools
+    - six
     - tornado >=4.3
+    - yaml
 
   run:
     - python
@@ -46,47 +46,45 @@ requirements:
 
 test:
   requires:
+    # jinja2 installed by bokeh
+    # packaging installed by bokeh
+    # pyyaml installed by bokeh
+
     # docs
     - colorcet
     - pygments
     - sphinx >=1.6
-    # pyyaml installed by bokeh
-    # packaging installed by bokeh
 
     # tests
+    - beautifulsoup4
+    - boto
     - colorama
     - coverage
     - flake8
     - lxml
-    - networkx
     - mock
-    - packaging
+    - networkx
+    - nodejs >=8.8.0,<9.0
+    - numba
     - pandas
+    - pillow
     - pytest >=3.3.0
     - pytest-cov >=1.8.1
-    - pytest-selenium >=1.6.0 [unix]
     - pytest-html
-    - selenium >=3
     - pytest-xdist
     - requests >=1.2.3
-    - beautifulsoup4
+    - selenium >=3
     - scipy
-    - numba
-    - pillow
-    - boto
-    - nodejs >=8.8.0,<9.0
-    # pyyaml installed by bokeh
-    # jinja2 installed by bokeh
 
     # examples
-    - notebook
-    - sympy
-    - scikit-learn
-    - networkx
-    - icalendar
     - flask >=1.0
+    - icalendar
+    - networkx
+    - notebook
     - pscript
     - pyshp
+    - scikit-learn
+    - sympy
 
   commands:
     - bokeh -h

conftest.py

@@ -7,24 +7,24 @@
 
 # Unfortunately these seem to all need to be centrally defined at the top level
 def pytest_addoption(parser):
+
+    # plugins/bokeh_server
     parser.addoption(
-        "--upload", dest="upload", action="store_true", default=False, help="upload test artefacts to S3"
+        "--bokeh-port", dest="bokeh_port", type=int, default=5006, help="port on which Bokeh server resides"
     )
+
+    # plugins/jupyter_notebook
     parser.addoption(
-        "--examples-log-file", dest="log_file", metavar="path", action="store", default='examples.log', help="where to write the complete log"
+        "--notebook-port", type=int, default=6007, help="port on which Jupyter Notebook server resides"
     )
 
-
-
+    # plugins/example_report
     parser.addoption(
-        "--bokeh-port", dest="bokeh_port", type=int, default=5006, help="port on which Bokeh server resides"
+        "--upload", dest="upload", action="store_true", default=False, help="upload test artefacts to S3"
     )
-
-
-
     parser.addoption(
-        "--output-cells", type=str, choices=['complain', 'remove', 'ignore'], default='complain',
-        help="what to do with notebooks' output cells")
+        "--examples-log-file", dest="log_file", metavar="path", action="store", default='examples.log', help="where to write the complete log"
+    )
     parser.addoption(
         "--report-path", action='store', dest='report_path', metavar='path', default='report.html',
         help='create examples html report file at given path.')
@@ -37,15 +37,3 @@ def pytest_addoption(parser):
     parser.addoption(
         "--no-js", action="store_true", default=False,
         help="only run python code and skip js and image diff")
-
-
-
-    parser.addoption(
-        "--notebook-port", type=int, default=6007, help="port on which Jupyter Notebook server resides"
-    )
-
-
-
-    parser.addoption(
-        "--set-new-base-screenshot", dest="set_new_base_screenshot", action="store_true", default=False,
-        help="Use to set a new screenshot for imagediff testing. Be sure to only set for the tests you want by usign the -k pytest option to select your test.")

scripts/ci/test.codebase

@@ -13,4 +13,4 @@ popd
 
 py.test -m sampledata
 
-py.test -m quality
+py.test -m codebase

setup.cfg

@@ -18,13 +18,12 @@ max-line-length = 165
 [tool:pytest]
 norecursedirs = build _build node_modules
 python_files = *_tests.py *_test.py test_*.py
-selenium_exclude_debug = html logs
 implicit_marker = unit
 markers =
     examples: an examples image-diff test
-    integration: an integration test that runs on Selenium
+    integration: an integration test
     js: a javascript test
-    quality: a code quality test
+    codebase: a test for codebase quality or policy
     sampledata: a test for bokeh.sampledata
     selenium: a test as requiring selenium
     unit: a python unit test (implicitly assigned for tests otherwise unmarked)