setup.py: support for "python setup.py test"

setup.py also got some minor cleanups

CHECKLIST.txt

@@ -1,13 +1,16 @@
 Release checklist
 -----------------
-0a. Ensure tests are passing: nosetests-2.7 --with-doctest -dvx python/tests/
-0b. shell> flake8 python/bllipparser/ParsingShell.py \
-                  python/bllipparser/RerankingParser.py \
-                  python/bllipparser/RerankerFeatureCorpus.py \
-                  python/bllipparser/Utility.py \
-                  python/bllipparser/__main__.py \
-                  setup.py
-    (we currently set flake8's ignore = E301,E302,E261)
+0.  Ensure tests are passing:
+        python setup.py test
+    Or manually:
+        nosetests-2.7 --with-doctest -dvx python/tests/
+        flake8 python/bllipparser/ParsingShell.py \
+               python/bllipparser/RerankingParser.py \
+               python/bllipparser/RerankerFeatureCorpus.py \
+               python/bllipparser/Utility.py \
+               python/bllipparser/__main__.py \
+               setup.py
+        (we currently set flake8's ignore = E301,E302,E261)
 1.  Version bumps in python/bllipparser/__init__.py and setup.py
 2.  Update READMEs
 3.  Sync README-python.rst content with python/bllipparser/__init__.py

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
-from distutils.core import setup, Extension
+from __future__ import print_function
+from distutils.core import setup, Command, Extension
 import os
 import subprocess
 from os.path import join, exists
@@ -10,7 +11,7 @@
 # outputs of these commands so you can build the Python modules without
 # these dependencies.
 
-def run(args):
+def run(*args):
     cmd = ' '.join(map(str, args))
     print("Running %r" % cmd)
     message = None
@@ -27,9 +28,27 @@ def run(args):
         raise SystemExit("Error while running command: %s\nBuild failed!" %
                          message)
 
-parser_base = 'first-stage/PARSE/'
-parser_wrapper = 'swig/wrapper.C'
-parser_wrapper_full = join(parser_base, parser_wrapper)
+class Test(Command):
+    user_options = []
+    def initialize_options(self):
+        pass
+    def finalize_options(self):
+        pass
+    def run(self):
+        python_dir = 'python/bllipparser'
+        # don't flake8 autogenerated files and a couple others with weird
+        # data or docstrings in them (ModelFetcher should ultimately be
+        # split from its data and checked again)
+        excluded_from_checking = ('CharniakParser.py', 'JohnsonReranker.py',
+                                  '__init__.py', 'ModelFetcher.py')
+        python_files = os.listdir(python_dir)
+        python_files = [join(python_dir, filename) for filename in python_files
+                        if filename not in excluded_from_checking
+                        and filename.endswith('.py')]
+        python_files.append('setup.py')
+
+        run('flake8', *python_files)
+        run('nosetests-2.7', '-dvx', '--with-doctest', 'python/tests')
 
 def is_newer(filename1, filename2):
     """Returns True if filename1 has a newer modification time than
@@ -49,17 +68,26 @@ def maybe_run_swig(wrapper_filename, module_name, base_directory,
             return
 
     print('Generating ' + module_name + ' SWIG wrapper files')
-    run(['swig', '-python', '-c++', '-module',
-         module_name, '-I' + base_directory,
-         '-Wall', '-classic', '-outdir', 'python/bllipparser',
-         '-o', wrapper_filename, swig_filename])
+    run('swig', '-python', '-c++', '-module', module_name,
+        '-I' + base_directory, '-Wall', '-classic',
+        '-outdir', 'python/bllipparser',
+        '-o', wrapper_filename, swig_filename)
+
+wrapper_path = 'swig/wrapper.C'
+
+#
+# parser wrapper
+#
+
+parser_base = 'first-stage/PARSE/'
+parser_wrapper_full = join(parser_base, wrapper_path)
 
 # generate parser SWIG files if needed
 maybe_run_swig(parser_wrapper_full, 'CharniakParser', parser_base,
                extra_deps=[join(parser_base, 'SimpleAPI.' + suffix)
                            for suffix in 'Ch'])
 
-parser_sources = (parser_wrapper, 'Bchart.C', 'BchartSm.C', 'Bst.C',
+parser_sources = (wrapper_path, 'Bchart.C', 'BchartSm.C', 'Bst.C',
                   'FBinaryArray.C', 'CntxArray.C', 'ChartBase.C',
                   'ClassRule.C', 'ECArgs.C', 'Edge.C', 'EdgeHeap.C',
                   'ExtPos.C', 'Feat.C', 'Feature.C', 'FeatureTree.C',
@@ -76,22 +104,25 @@ def maybe_run_swig(wrapper_filename, module_name, base_directory,
                           sources=parser_sources, include_dirs=[parser_base],
                           libraries=['stdc++'])
 
+#
+# reranker wrapper
+#
+
 reranker_base = 'second-stage/programs/features/'
-reranker_wrapper = 'swig/wrapper.C'
-reranker_wrapper_full = reranker_base + reranker_wrapper
+reranker_wrapper_full = reranker_base + wrapper_path
 reranker_read_tree = 'read-tree.cc'
-reranker_read_tree_full = reranker_base + 'read-tree.cc'
+reranker_read_tree_full = reranker_base + reranker_read_tree
 
 # generate reranker SWIG files if needed
 maybe_run_swig(reranker_wrapper_full, 'JohnsonReranker', reranker_base)
 
 # generate reranker tree reader if needed
 if not exists(reranker_read_tree_full):
-    run(['flex', '-o' + reranker_read_tree_full,
-        reranker_read_tree_full.replace('.cc', '.l')])
+    run('flex', '-o' + reranker_read_tree_full,
+        reranker_read_tree_full.replace('.cc', '.l'))
 
 reranker_sources = [join(reranker_base, src) for src in
-                    (reranker_wrapper, 'simple-api.cc', 'heads.cc',
+                    (wrapper_path, 'simple-api.cc', 'heads.cc',
                      reranker_read_tree, 'sym.cc')]
 
 reranker_module = Extension('bllipparser._JohnsonReranker',
@@ -119,4 +150,5 @@ def maybe_run_swig(wrapper_filename, module_name, base_directory,
       platforms=['POSIX'],
       ext_modules=[parser_module, reranker_module],
       packages=['bllipparser'],
-      package_dir={'bllipparser': 'python/bllipparser'})
+      package_dir={'bllipparser': 'python/bllipparser'},
+      cmdclass={'test': Test})