Simplify testing (#124)

* add conftest to build messages and set path prior to collection/running tests

* clean up tests

* update README for testing changes

* edits

* some fixes and version bump

* only write generated cython code if it differs from what's there (to avoid unnecessary compilation)

* add clean argument to compiler to force recompilation of messages

README.md

@@ -28,7 +28,7 @@ Cython to work properly is the trickiest bit especially if you are still using
 ### Contributing
 
 People use protobuf in many different ways. Pyrobuf handles the use cases of
-AppNexus and other contributors, but is not yet a 100% shoe-in replacement to
+AppNexus and other contributors, but is not yet a 100% drop-in replacement to
 what protoc would generate.
 
 You can help make it so!
@@ -37,46 +37,42 @@ Fork and clone the repository, then run:
 
     $ python setup.py develop
 
-It will generate the platform specific pyrobuf_list then compile
-the pyrobuf_list and pyrobuf_util modules.
+It will generate the platform specific `pyrobuf_list` then compile
+the `pyrobuf_list` and `pyrobuf_util` modules.
 
 
 ### Unit Testing
 
-First you have to build the `.proto` files inside the `tests` folder:
+You can run the test suite (a work in progress) using py.test directly:
 
-        pyrobuf --install tests/proto/
-
-
-Now you can run the test suite (a work in progress) using py.test directly:
-
-    $ PYTHONPATH=. py.test
+    $ py.test
 
 Or using the `test` command (which installs pytest if not already available):
 
     $ python setup.py test
 
+Either method will automatically build all the protobuf message specs in
+`tests/proto` and point the `PYTHONPATH` to the built messages before running
+the tests.
+
 Re-running the `develop` or `test` commands will automatically re-build the
-pyrobuf_list and pyrobuf_util modules if necessary.
+`pyrobuf_list` and `pyrobuf_util` modules if necessary.
 
 The `clean` command does the house keeping for you:
 
     $ python setup.py clean
 
-`test__gen_message` will attempt to process all the proto files in
-`tests/proto`.
-
 If you find that pyrobuf does not work for one of your proto files, add a minimal
 proto file to `tests/proto` that breaks before submitting a pull request.
 
 Pull requests including a breaking test are gold!
 
-Improving testing is on the cards.
+Improving testing is in the cards.
 
 
 ### Installation
 
-You may very well be able to just use pyrobuf as is ... just pip it!
+You may very well be able to just use pyrobuf as is... just pip it!
 
 ```
 $ pip install pyrobuf

pyrobuf/compile.py

@@ -24,14 +24,16 @@ class Compiler(object):
     t_pyx = _env.get_template('proto_pyx.tmpl')
 
     def __init__(self, sources, out="out", build="build", install=False,
-                 proto3=False, force=False, package=None, includes=None):
+                 proto3=False, force=False, package=None, includes=None,
+                 clean=False):
         self.sources = sources
         self.out = out
         self.build = build
         self.install = install
         self.force = force
         self.package = package
         self.includes = includes or []
+        self.clean = clean
         here = os.path.dirname(os.path.abspath(__file__))
         self.include_path = [os.path.join(here, 'src'), self.out]
         self._generated = set()
@@ -63,12 +65,16 @@ def parse_cli_args(cls):
                             help="force install")
         parser.add_argument('--package', type=str, default=None,
                             help="name of package to compile to")
-        parser.add_argument('--include', action='append')
+        parser.add_argument('--include', action='append',
+                            help="add directory to includes path")
+        parser.add_argument('--clean', action='store_true',
+                            help="force recompilation of messages")
         args = parser.parse_args()
 
         return cls(args.sources, out=args.out_dir, build=args.build_dir,
                    install=args.install, proto3=args.proto3, force=args.force,
-                   package=args.package, includes=args.include)
+                   package=args.package, includes=args.include,
+                   clean=args.clean)
 
     def compile(self):
         script_args = ['build', '--build-base={0}'.format(self.build)]
@@ -150,11 +156,30 @@ def _write(self, name, msg_def):
         name_pyx = "{}_proto.pyx".format(name)
         self._pyx_files.append(os.path.join(self.out, name_pyx))
 
-        with open(os.path.join(self.out, name_pxd), 'w') as fp:
-            fp.write(self.t_pxd.render(msg_def, version_major=_VM))
-
-        with open(os.path.join(self.out, name_pyx), 'w') as fp:
-            fp.write(self.t_pyx.render(msg_def, version_major=_VM))
+        generated_pxd = self.t_pxd.render(msg_def, version_major=_VM)
+        generated_pyx = self.t_pyx.render(msg_def, version_major=_VM)
+        write_pxd = True
+        write_pyx = True
+
+        if not self.clean and os.path.exists(os.path.join(self.out, name_pxd)):
+            with open(os.path.join(self.out, name_pxd), 'r') as fp:
+                if fp.read().strip() == generated_pxd.strip():
+                    write_pxd = False
+                    print('{} has not changed'.format(name_pxd))
+
+        if not self.clean and os.path.exists(os.path.join(self.out, name_pyx)):
+            with open(os.path.join(self.out, name_pyx), 'r') as fp:
+                if fp.read().strip() == generated_pyx.strip():
+                    write_pyx = False
+                    print('{} has not changed'.format(name_pyx))
+
+        if write_pxd:
+            with open(os.path.join(self.out, name_pxd), 'w') as fp:
+                fp.write(generated_pxd)
+
+        if write_pyx:
+            with open(os.path.join(self.out, name_pyx), 'w') as fp:
+                fp.write(generated_pyx)
 
     def _package(self):
         meta = {'imports': [], 'messages': [], 'enums': []}

setup.py

@@ -10,7 +10,7 @@
 import sys
 
 
-VERSION = "0.8.4"
+VERSION = "0.8.5"
 HERE = os.path.dirname(os.path.abspath(__file__))
 PYROBUF_DEFS_PXI = "pyrobuf_defs.pxi"
 PYROBUF_LIST_PXD = "pyrobuf_list.pxd"

tests/conftest.py

@@ -0,0 +1,26 @@
+import os
+import sys
+
+from distutils.util import get_platform
+from pyrobuf.compile import Compiler
+
+
+def pytest_sessionstart(session):
+    # Build test messages from proto specs
+    here = os.path.dirname(os.path.abspath(__file__))
+    proto = [os.path.join(here, 'proto', filename)
+             for filename in os.listdir(os.path.join(here, 'proto'))]
+    compiler = Compiler(proto, out='tests/out', build='tests/build')
+    compiler.compile()
+
+    # Add test directory into path
+    if here not in sys.path:
+        sys.path.append(here)
+
+    # Insert built messages into path
+    build = os.path.join(here, 'build')
+    lib_path = os.path.join(build, "lib.{0}-{1}".format(get_platform(),
+                                                        sys.version[0:3]))
+
+    if lib_path not in sys.path:
+        sys.path.insert(0, lib_path)

tests/perf_lists.py

@@ -3,12 +3,13 @@
 
 from pyrobuf_list import *
 
+
 def main():
     start = time.time()
     x = []
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(float(i))
 
     for i in x:
@@ -21,7 +22,7 @@ def main():
     x = TypedList(float)
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(float(i))
 
     for i in x:
@@ -34,7 +35,7 @@ def main():
     x = DoubleList()
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(float(i))
 
     for i in x:
@@ -47,7 +48,7 @@ def main():
     x = FloatList()
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(float(i))
 
     for i in x:
@@ -60,7 +61,7 @@ def main():
     x = array('f')
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(float(i))
 
     for i in x:
@@ -82,13 +83,11 @@ def main():
     end = time.time()
     print("array 'd' took %f seconds" % (end - start))
 
-    print ""
-
     start = time.time()
     x = []
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(i)
 
     for i in x:
@@ -101,7 +100,7 @@ def main():
     x = IntList()
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(i)
 
     for i in x:
@@ -114,7 +113,7 @@ def main():
     x = Int32List()
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(i)
 
     for i in x:
@@ -127,7 +126,7 @@ def main():
     x = array('i')
     y = 0
 
-    for i in xrange(1000000):
+    for i in range(1000000):
         x.append(i)
 
     for i in x:

tests/test_deprecated_field.py

@@ -1,11 +1,7 @@
 import unittest
 import warnings
 
-import pytest
-from proto_lib_fixture import proto_lib
 
-
-@pytest.mark.usefixtures('proto_lib')
 class DecimalDefaultsTest(unittest.TestCase):
     def setUp(self):
         from test_message_field_types_proto import TestDeprecatedField

tests/test_field_defaults.py

@@ -1,14 +1,10 @@
 import unittest
 
-import pytest
-from proto_lib_fixture import proto_lib
-
 
 TestDecimalDefaultsMessage = None
 TestStringDefaultsMessage = None
 
 
-@pytest.mark.usefixtures('proto_lib')
 class DecimalDefaultsTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -61,7 +57,6 @@ def test_default_of_one_e_minus_6(self):
         self.assertEqual(self.message.one_e_minus_6, 1e-6)
 
 
-@pytest.mark.usefixtures('proto_lib')
 class StringDefaultsTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):

tests/test_has_field.py

@@ -1,16 +1,12 @@
 import unittest
 
-import pytest
-from proto_lib_fixture import proto_lib
-
 
 Test = None
 TestRef = None
 TestSs1 = None
 TestSs1Thing = None
 
 
-@pytest.mark.usefixtures('proto_lib')
 class HasFieldTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):

tests/test_imported_enums.py

@@ -1,8 +1,5 @@
 import unittest
 
-import pytest
-from proto_lib_fixture import proto_lib
-
 
 # These can't be imported until the test_imported_enums_proto module has been built.
 CLOSE = None
@@ -11,7 +8,6 @@
 UsesImportedEnumsMessage = None
 
 
-@pytest.mark.usefixtures('proto_lib')
 class ImportedEnumsTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):

tests/test_is_initialized.py

@@ -1,15 +1,11 @@
 import unittest
 
-import pytest
-from proto_lib_fixture import proto_lib
-
 
 TestIsInitialized = None
 SubMessage = None
 TestWithRequiredSubMessage = None
 
 
-@pytest.mark.usefixtures('proto_lib')
 class MergeFromTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):