Change scripts test hierarchy

.pylintrc

@@ -30,7 +30,7 @@ ignore=CVS
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
-ignore-patterns=
+ignore-patterns=test_.*.py
 
 # Pickle collected data for later comparisons.
 persistent=yes

Jenkinsfile

@@ -6,6 +6,7 @@ stage("Sanity Check") {
       conda env update -f env/pylint.yml
       source activate gluon_nlp_pylint
       conda list
+      make clean
       make pylint
       """
     }
@@ -23,8 +24,9 @@ stage("Unit Test") {
         conda list
         python -m spacy download en
         python -m nltk.downloader all
+        make clean
         python setup.py install
-        nosetests -v --nocapture --with-timer tests/unittest
+        nosetests -v --nocapture --with-timer scripts/*/test_*.py tests/unittest
         """
       }
     }
@@ -39,8 +41,9 @@ stage("Unit Test") {
         conda list
         python -m spacy download en
         python -m nltk.downloader all
+        make clean
         python setup.py install
-        nosetests -v --nocapture --with-timer tests/unittest
+        nosetests -v --nocapture --with-timer scripts/*/test_*.py tests/unittest
         """
       }
     }
@@ -57,7 +60,7 @@ stage("Deploy") {
       conda list
       python setup.py install
       export LD_LIBRARY_PATH=/usr/local/cuda/lib64
-      make -C docs clean
+      make clean
       make -C docs html"""
 
       if (env.BRANCH_NAME.startsWith("PR-")) {
@@ -74,4 +77,4 @@ stage("Deploy") {
       }
     }
   }
-}
+}

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include gluonnlp *.py
+include LICENSE
+include README.rst
+recursive-exclude tests *
+recursive-exclude scripts *

Makefile

@@ -23,8 +23,12 @@ pylint:
 docs:
 	make -C docs html
 
+clean:
+	rm -rf gluonnlp.egg-info build dist | true
+	make -C docs clean
+
 test:
-	nosetests -v --nocapture tests
+	nosetests -v --nocapture --with-timer scripts/*/test_*.py tests/unittest
 
 release:
 	python setup.py sdist

README.rst

@@ -0,0 +1,5 @@
+Gluon NLP Toolkit
+=================
+
+This is a toolkit that enables fast prototyping for NLP research and
+applications.

env/py2.yml

@@ -4,10 +4,11 @@ channels:
   - defaults
 dependencies:
   - python=2.7
+  - perl
   - sphinx=1.7.2
   - spacy
   - nltk
-  - nose
+  - nose>=1.3.7
   - nose-timer
   - pip:
     - mxnet>=1.2.0b20180415

env/py3.yml

@@ -4,10 +4,11 @@ channels:
   - defaults
 dependencies:
   - python=3.6
+  - perl
   - sphinx=1.7.2
   - spacy
   - nltk
-  - nose
+  - nose>=1.3.7
   - nose-timer
   - pip:
     - mxnet>=1.2.0b20180415

scripts/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=wildcard-import
+"""NLP examples."""

scripts/language_model/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=wildcard-import
+"""Language Model example."""

scripts/nmt/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=wildcard-import
+"""NMT example."""

scripts/sentiment_analysis/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=wildcard-import
+"""Sentiment Analysis example."""

scripts/tests/__init__.py

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=wildcard-import
+"""NLP example tests."""

scripts/nmt/test_bleu.py → scripts/tests/test_bleu.py

@@ -27,7 +27,7 @@
 import codecs
 import numpy as np
 from numpy.testing import assert_allclose
-from bleu import compute_bleu
+from ..nmt.bleu import compute_bleu
 
 
 actions = ['deletion', 'replacement', 'add']
@@ -94,7 +94,6 @@ def _write_reference(references, path='reference'):
 
 
 def test_bleu():
-    """Test BLEU utility against multi-bleu.perl."""
     n = 100
     max_len = 50
     n_refs = 5

setup.py

@@ -24,7 +24,7 @@ def find_version(*file_paths):
     raise RuntimeError("Unable to find version string.")
 
 
-readme = open('README.md').read()
+readme = open('README.rst').read()
 
 VERSION = find_version('gluonnlp', '__init__.py')
 
@@ -49,5 +49,6 @@ def find_version(*file_paths):
         'scripts',
     )),
     zip_safe=True,
+    include_package_data=True,
     install_requires=requirements,
 )