fix mars/tensor coverage issue by applying different configurations (#41

)

* fix mars/tensor coverage issue by applying different configurations

* add fail-under & fix get_next_port

.coveragerc

@@ -1,6 +1,8 @@
 [run]
 branch = True
+cover_pylib = False
 concurrency = multiprocessing,gevent
+parallel = True
 include =
     mars/*
 omit =

.coveragerc-tensor

@@ -0,0 +1,19 @@
+# this configuration is for mars.tensor module as
+# coverage.py currently doesn't support measuring
+# gevent with threading.
+
+[run]
+branch = True
+cover_pylib = False
+parallel = True
+include =
+    mars/*
+omit =
+    mars/lib/functools32/*
+    mars/lib/futures/*
+    mars/lib/enum.py
+    mars/lib/six.py
+    mars/lib/tblib/*
+    mars/lib/uhashring/*
+    mars/serialize/protos/*
+    */tests/*

.travis.yml

@@ -36,7 +36,13 @@ install:
 
 # command to run tests
 script:
-  pytest --timeout=1500 --forked --cov-config .coveragerc --cov=mars mars
+  mkdir -p build &&
+  pytest --cov-report= --cov-config .coveragerc-tensor --cov=mars --timeout=1500 mars/tensor &&
+  mv .coverage build/.coverage.tensor.file &&
+  pytest --cov-report= --cov-config .coveragerc --cov=mars --timeout=1500 --forked
+    $(find mars -maxdepth 1 -mindepth 1 -type d -not -path "mars/tensor" -not -path "*__pycache__*") &&
+  mv .coverage build/.coverage.main.file &&
+  coverage combine build/ && coverage report --fail-under=80
 
 after_success:
   - coveralls

README.rst

@@ -1,6 +1,8 @@
 Mars
 ====
 
+|PyPI version| |Build| |Coverage| |Docs| |License| |Implementation|
+
 Mars is a tensor-based unified framework for large-scale data computation.
 
 Mars tensor
@@ -53,3 +55,15 @@ Easy to scale in and scale out
 Mars can scale in to a single machine, and scale out to a cluster with thousands of machines.
 Both the local and distributed version share the same piece of code,
 it's fairly simple to migrate from a single machine to a cluster due to the increase of data.
+
+.. |Build| image:: https://img.shields.io/travis/mars-project/mars.svg?style=flat-square
+   :target: https://travis-ci.org/mars-project/mars
+.. |Coverage| image:: https://img.shields.io/coveralls/github/mars-project/mars.svg?style=flat-square
+   :target: https://coveralls.io/github/mars-project/mars
+.. |PyPI version| image:: https://img.shields.io/pypi/v/pymars.svg?style=flat-square
+   :target: https://pypi.python.org/pypi/pymars
+.. |Docs| image:: https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat-square
+   :target: http://mars-project.readthedocs.org/
+.. |License| image:: https://img.shields.io/pypi/l/pymars.svg?style=flat-square
+   :target: https://github.com/mars-project/mars/blob/master/LICENSE
+.. |Implementation| image:: https://img.shields.io/pypi/implementation/pymars.svg?style=flat-square

bin/travis-upload.sh

@@ -1,21 +1,24 @@
 #!/bin/bash
-python setup.py sdist --formats=gztar
-python setup.py bdist_wheel
 
-for whl in dist/*.whl; do
+if [ "$TRAVIS_TAG" ]; then
+  echo "[distutils]"                                  > ~/.pypirc
+  echo "index-servers ="                             >> ~/.pypirc
+  echo "    pypi"                                    >> ~/.pypirc
+  echo "[pypi]"                                      >> ~/.pypirc
+  echo "repository=https://upload.pypi.org/legacy/"  >> ~/.pypirc
+  echo "username=pyodps"                             >> ~/.pypirc
+  echo "password=$PASSWD"                            >> ~/.pypirc
+
+  python -m pip install twine
+
+  python setup.py sdist --formats=gztar
+  python setup.py bdist_wheel
+
+  for whl in dist/*.whl; do
 	auditwheel repair $whl -w dist/
-done
-rm dist/*-linux*.whl
+  done
+  rm dist/*-linux*.whl
 
-echo "[distutils]"                                  > ~/.pypirc
-echo "index-servers ="                             >> ~/.pypirc
-echo "    pypi"                                    >> ~/.pypirc
-echo "[pypi]"                                      >> ~/.pypirc
-echo "repository=https://upload.pypi.org/legacy/"  >> ~/.pypirc
-echo "username=pyodps"                             >> ~/.pypirc
-echo "password=$PASSWD"                            >> ~/.pypirc
-python -m pip install twine
-if [ "$TRAVIS_TAG" ]; then
   python -m twine upload -r pypi --skip-existing dist/*.tar.gz;
 else
   echo "Not on a tag, won't deploy to pypi";

mars/distributor.py

@@ -15,9 +15,9 @@
 import hashlib
 import logging
 
-from mars.compat import six, functools32
-from mars.utils import to_binary
-from mars.actors import Distributor
+from .compat import six, functools32
+from .utils import to_binary
+from .actors import Distributor
 
 logger = logging.getLogger(__name__)
 

mars/lib/gipc.pyx

@@ -71,6 +71,11 @@ try:
 except ImportError:
     import pickle
 
+try:
+    from pytest_cov.embed import cleanup_on_sigterm
+except ImportError:
+    cleanup_on_sigterm = lambda: None
+
 cdef:
     bint WINDOWS, PY2
 
@@ -95,7 +100,7 @@ if WINDOWS:
 
 # Logging for debugging purposes. Usage of logging in this simple form in the
 # context of multiple processes might yield mixed messages in the output.
-log = logging.getLogger(__name__)
+log = logging.getLogger('gipc')
 
 
 class GIPCError(Exception):
@@ -1121,6 +1126,7 @@ def _reset_signal_handlers():
         # in the signal module.
         if s < signal.NSIG:
             signal.signal(s, signal.SIG_DFL)
+    cleanup_on_sigterm()
 
 
 PY3 = sys.version_info[0] == 3

mars/lib/sparse/coo.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import numpy as np
-from mars.lib.sparse.array import SparseNDArray
+from .array import SparseNDArray
 from .. import six
 
 

mars/node_info.py

@@ -20,8 +20,11 @@
 
 logger = logging.getLogger(__name__)
 
-from mars.utils import git_info, readable_size
-from mars.actors import FunctionActor
+from . import resource
+from .utils import git_info, readable_size
+from .actors import FunctionActor
+from .compat import six
+
 
 try:
     import numpy as np
@@ -36,9 +39,6 @@
 except ImportError:
     cp = None
 
-from mars import resource
-from mars.compat import six
-
 _collectors = dict()
 
 

mars/utils.py

@@ -24,6 +24,7 @@
 import os
 import random
 import socket
+import struct
 import subprocess
 import sys
 import time
@@ -319,12 +320,14 @@ def get_next_port(typ=None):
                     break
         p.stdout.close()
 
-    idx = random.randint(0, HIGH_PORT_BOUND - LOW_PORT_BOUND - len(occupied))
+    randn = struct.unpack('<Q', os.urandom(8))[0]
+    idx = int(randn % (1 + HIGH_PORT_BOUND - LOW_PORT_BOUND - len(occupied)))
     for i in irange(LOW_PORT_BOUND, HIGH_PORT_BOUND + 1):
+        if i in occupied:
+            continue
         if idx == 0:
             return i
-        elif i not in occupied:
-            idx -= 1
+        idx -= 1
     raise SystemError('No ports available.')