Merge branch 'release/0.16.0'

.travis.yml

@@ -16,7 +16,6 @@ install:
   - travis_retry pip install --upgrade pip
   - travis_retry pip install wheel==0.26.0
   - travis_retry pip install invoke==0.11.1
-  - sed 's/^rpy2==/#rpy2==/' -i requirements.txt
   - travis_retry invoke wheelhouse --develop
   - travis_retry invoke install --develop
 

CHANGELOG

@@ -2,6 +2,11 @@
 ChangeLog
 *********
 
+0.16.0 (2016-09-13)
+===================
+- Feature: MFR now does .sav conversion via pspp-convert instead of rpy2.
+- Fix: Update ipython notebook renderer to use split-out nbconvert and nbformat libraries.
+
 0.15.0 (2016-08-25)
 ===================
 - Feature: add analytics to MFR requests.  MFR now keeps track of requests, handlers, renderers,

Dockerfile

@@ -22,6 +22,9 @@ RUN apt-get update \
     # unoconv dependencies
     && apt-get install -y \
         unoconv \
+    # pspp dependencies
+    && apt-get install -y \
+        pspp \
     && apt-get clean \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*

README.md

@@ -6,21 +6,34 @@
 
 A Python package for rendering files to HTML via an embeddable iframe.
 
-### Startup commands
+### Documentation
+
+*Note: https://readthedocs.org/ is currently unable to build documentation for Python 3.5 projects.*  The documentation available at https://mfr.readthedocs.io/en/latest/ is outdated (v0.15.0). For the most up-to-date documentation, build locally. Within your checkout, run:
+
+```bash
+pip install -r doc-requirements.txt
+cd docs
+make
+open _build/html/index.html
+```
+
+### Setting up
 
 Install the latest version of python3.5.
 
 For MacOSX users:
 
 ```bash
 brew install python3
-brew install r
+# optional, needed for some converters
+brew install pspp unoconv
 ```
 For Ubuntu users:
 
 ```bash
 apt-get install python3
-apt-get install r-base
+# optional, needed for some converters
+apt-get install pspp unoconv
 ```
 
 After installing python3.5, create the virtual environment with the following commands:
@@ -30,12 +43,19 @@ pip install virtualenv
 pip install virtualenvwrapper
 mkvirtualenv --python=`which python3.5` mfr
 pip install invoke==0.11.1
-invoke install --develop
+invoke install
 invoke server
 ```
-### Testing Configuration (optional)
 
-MFR looks for local configuration in `~/.cos/mfr-test.json`.  The defaults should suffice for most local testing.  If you're running the OSF on something other than `http://localhost:5000/`, you'll need to update the `ALLOWED_PROVIDER_DOMAINS` settings value:
+### Configuring
+
+MFR configuration is done through a JSON file (`mfr-test.json`) that lives in the `.cos` directory of your home directory.  If this is your first time setting up MFR or its sister project, [WaterButler](https://github.com/CenterForOpenScience/waterbutler/), you probably do not have this directory and will need to create it:
+
+```bash
+mkdir ~/.cos
+```
+
+The defaults should suffice for most local testing.  If you're running the OSF on something other than `http://localhost:5000/`, you'll need to update the `ALLOWED_PROVIDER_DOMAINS` settings value:
 
 ```json
 {
@@ -45,7 +65,7 @@ MFR looks for local configuration in `~/.cos/mfr-test.json`.  The defaults shoul
 }
 ```
 
-If you encounter the error message `TypeError: throw() takes 2 positional arguments but 4 were given`, you've run into a [core asyncio bug](https://bugs.python.org/issue25394)!  This bug is triggered by turning on debugging. You'll need to set `SERVER_CONFIG.DEBUG` to `false` in your `mfr-test.json`, e.g.
+If you encounter the error message `TypeError: throw() takes 2 positional arguments but 4 were given`, you've run into a [core asyncio bug](https://bugs.python.org/issue25394)!  This bug is triggered by turning on debugging. You'll need to set the `SERVER_CONFIG.DEBUG` flag to `false`:
 
 ```json
 {
@@ -55,11 +75,20 @@ If you encounter the error message `TypeError: throw() takes 2 positional argume
 }
 ```
 
+### Testing
+
+Before running the tests, you will need to install some additional requirements. In your checkout, run:
+
+```bash
+invoke install --develop
+invoke test
+```
+
 ### Create your own module
 
 Interested in adding support for a new provider or file format? Check out the CONTRIBUTING.rst docs.
 
-### License 
+### License
 
 Copyright 2013-2016 Center for Open Science
 
@@ -75,6 +104,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-### COS is Hiring!
+### COS is hiring!
 
-Want to help save science? Want to get paid to develop free, open source software? [Check out our openings!](http://cos.io/jobs)
+Want to help save science? Want to get paid to develop free, open source software? [Check out our openings!](https://cos.io/jobs/)

docs/install.rst

@@ -14,7 +14,7 @@ Or download one of the following:
 * tarball_
 * zipball_
 
-Make sure that you have installed R, are using python3.5, and have installed invoke for your current python3 version.
+Make sure that you have installed pspp, are using python3.5, and have installed invoke for your current python3 version.
 
 Install the version of invoke found in the requirements.txt file. Currently 0.11.1
 

mfr/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.15.0'
+__version__ = '0.16.0'
 __import__('pkg_resources').declare_namespace(__name__)

mfr/extensions/ipynb/render.py

@@ -1,10 +1,10 @@
 import os
 
-import IPython
-from IPython import nbformat
-from IPython.config import Config
-from IPython.nbconvert.exporters import HTMLExporter
+import nbformat
+import nbconvert
+from traitlets.config import Config
 from mako.lookup import TemplateLookup
+from nbconvert.exporters import HTMLExporter
 
 from mfr.core import extension
 from mfr.extensions.ipynb import exceptions
@@ -19,7 +19,8 @@ class IpynbRenderer(extension.BaseRenderer):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.metrics.add('ipython_version', IPython.__version__)
+        self.metrics.add('nbformat_version', nbformat.__version__)
+        self.metrics.add('nbconvert_version', nbconvert.__version__)
 
     def render(self):
         try:

mfr/extensions/tabular/libs/__init__.py

@@ -23,6 +23,10 @@ def dta_pandas():
     return dta_pandas
 
 
+def sav_stdlib():
+    from ..libs.stdlib_tools import sav_stdlib
+    return sav_stdlib
+
 def sav_pandas():
     from ..libs.panda_tools import sav_pandas
     return sav_pandas

mfr/extensions/tabular/libs/panda_tools.py

@@ -1,7 +1,9 @@
+from tempfile import NamedTemporaryFile
+
 import numpy
 import pandas
-from tempfile import NamedTemporaryFile
-from ..utilities import header_population, strip_comments
+
+from mfr.extensions.tabular.utilities import header_population, strip_comments, sav_to_csv
 
 
 def csv_pandas(fp):
@@ -36,11 +38,14 @@ def dta_pandas(fp):
 
 
 def sav_pandas(fp):
-    """Read and convert a sav file to JSON format using the pandas library
+    """Read and convert a .sav file to a .csv file via pspp, then convert that to JSON format
+    using the pandas library
+
     :param fp: File pointer object
     :return: tuple of table headers and data
     """
-    dataframe = robjectify(fp)
+    csv_file = sav_to_csv(fp)
+    dataframe = pandas.read_csv(csv_file.name, low_memory=False)
     return data_from_dataframe(dataframe)
 
 
@@ -64,15 +69,3 @@ def data_from_dataframe(dataframe):
                 data_row[name] = value
         data.append(data_row)
     return {'Sheet 1': (header, data)}
-
-
-def robjectify(fp):
-    """Create a dataframe object using R"""
-
-    import pandas.rpy.common as common
-    import rpy2.robjects as robjects
-    r = robjects
-    r.r("require(foreign)")
-    r.r('x <- read.spss("{}",to.data.frame=T)'.format(fp.name))
-    r.r('row.names(x) = 0:(nrow(x)-1)')
-    return common.load_data('x')

mfr/extensions/tabular/libs/stdlib_tools.py

@@ -1,6 +1,8 @@
-import csv
 import re
+import csv
+
 from ..exceptions import EmptyTableException
+from mfr.extensions.tabular import utilities
 
 
 def csv_stdlib(fp):
@@ -42,6 +44,19 @@ def csv_stdlib(fp):
     return {'Sheet 1': (columns, rows)}
 
 
+def sav_stdlib(fp):
+    """Read and convert a .sav file to .csv with pspp, then convert that to JSON format using
+    the python standard library
+
+    :param fp: File pointer object to a .sav file
+    :return: tuple of table headers and data
+    """
+    csv_file = utilities.sav_to_csv(fp)
+    with open(csv_file.name, 'r') as file:
+        csv_file.close()
+        return csv_stdlib(file)
+
+
 def _set_dialect_quote_attrs(dialect, data):
     """Set quote-related dialect attributes based on up to 2kb of csv data.
 

mfr/extensions/tabular/settings.py

@@ -20,7 +20,7 @@
     '.xlsx': [libs.xlsx_xlrd],
     '.xls': [libs.xlsx_xlrd],
     '.dta': [libs.dta_pandas],
-    '.sav': [libs.sav_pandas],
+    '.sav': [libs.sav_stdlib],
     # '.ods': [libs.ods_ezodf],
 })
 
@@ -38,3 +38,5 @@
     'syncColumnCellResize': True,
     'multiColumnSort': True,
 })
+
+PSPP_CONVERT_BIN = config.get('PSPP_CONVERT_BIN', '/usr/bin/pspp-convert')

mfr/extensions/tabular/utilities.py

@@ -1,6 +1,9 @@
 import re
+import subprocess
+from tempfile import NamedTemporaryFile
 
-from mfr.extensions.tabular import compat
+from mfr.core import exceptions
+from mfr.extensions.tabular import compat, settings
 
 
 def header_population(headers):
@@ -35,3 +38,22 @@ def strip_comments(src, dest):
         data = data.encode('utf-8', 'ignore')
     dest.write(data)
     dest.seek(0)
+
+
+def sav_to_csv(fp):
+    """Converts a SPSS .sav to a .csv file by calling out to ``pspp-convert``.
+
+    :param fp: file pointer object to .sav file
+    :return: file pointer to .csv file. You are responsible for closing this.
+    """
+    csv_file = NamedTemporaryFile(mode='w+b', suffix='.csv')
+    try:
+        subprocess.check_call([
+            settings.PSPP_CONVERT_BIN,
+            fp.name,
+            csv_file.name,
+        ])
+    except subprocess.CalledProcessError:
+        raise exceptions.ExporterError(
+            'Unable to convert the SPSS file to CSV, please try again later.', code=400)
+    return csv_file

requirements.txt

@@ -22,7 +22,9 @@ pydocx==0.7.0
 Pillow==2.8.2
 
 # IPython
-ipython==3.1.0
+nbconvert==4.2.0
+nbformat==4.1.0
+traitlets==4.2.2
 jsonschema==2.4.0
 jinja2==2.7.3
 mistune==0.7
@@ -37,9 +39,6 @@ docutils==0.12
 pandas==0.17.1
 git+https://github.com/icereval/xlrd.git
 
-# Rpy
-rpy2==2.7.8
-
 # Md
 markdown==2.6.2