Merge branch 'release/0.24.0'

.mailmap

@@ -12,12 +12,14 @@ Dawn Pattison <pattison.dawn.r@gmail.com>
 Elijah Hamovitz <elijahhamovitz@gmail.com>
 Eric Bower <neurosnap@gmail.com>
 Erin Braswell <erin.braswell@gmail.com> erinspace <erin.braswell@gmail.com>
-Fitz Elliott <fitz@cos.io>
+Fitz Elliott <fitz@cos.io> Fitz Elliott <felliott@fiskur.org>
 Haoyu Chen <hc6as@virginia.edu> haoyuchen1992 <hc6as@virginia.edu>
 Jeffrey Spies <jspies@gmail.com> Jeffrey Spies <github@jeffspies.com>
+John Tordoff <john@cos.io> Johnetordoff <Johnetordoff@users.noreply.github.com>
 Jonathon Love <jon@thon.cc>
 Joshua Carp <jm.carp@gmail.com>
 Kurtis Jungersen <kmjungersen@gmail.com> Kurtis Jungersen <Kurtisjungersen@gmail.com>
+Longze Chen <cs.longze.chen@gmail.com>
 Lyndsy Simon <lyndsy@lyndsysimon.com>
 Matt Frazier <maf7sm@virginia.edu>mfraezz <maf7sm@virginia.edu>
 Matthew Keitelman <zamattiac@gmail.com>
@@ -29,8 +31,8 @@ Nezar Abdennur <nabdennur@gmail.com>
 Oliver M Daniel <olivermdaniel@gmail.com>
 Peter Fan <peter.j.fan@gmail.com>
 Rafael de Lucena Valle <rafaeldelucena@gmail.com>
+Stefanie Schirmer <sschirme@gmail.com>
 Steven Loria <sloria1@gmail.com> sloria <sloria1@gmail.com>
 Tom Baxter <tkb608@gmail.com> TomBaxter <tkb608@gmail.com>
 Tom Baxter <tkb608@gmail.com> tkb608 <tkb608@gmail.com>
-linse <sschirme@gmail.com>
 

AUTHORS.rst

@@ -10,12 +10,12 @@ Contributors
 - Austin Macdonald `@asmacdo <https://github.com/asmacdo>`_
 - Alex Schiller `@alexschiller <https://github.com/alexschiller>`_
 - Kurtis Jungersen `@kmjungersen <https://github.com/kmjungersen>`_
--  `@nvictus <https://github.com/nvictus>`_
+- Nezar Abdennur `@nvictus <https://github.com/nvictus>`_
 - Elijah Hamovitz `@Hamms <https://github.com/Hamms>`_
 - Tom Baxter `@TomBaxter <https://github.com/TomBaxter>`_
 - Joshua Carp `@jmcarp <https://github.com/jmcarp>`_
 - Eric Bower `@neurosnap <https://github.com/neurosnap>`_
--  `@linse <https://github.com/linse>`_
+- Stefanie Schirmer `@linse <https://github.com/linse>`_
 - Meli Lewis `@meli-lewis <https://github.com/meli-lewis>`_
 - Barrett Harber `@binoculars <https://github.com/binoculars>`_
 - Lyndsy Simon `@lyndsysimon <https://github.com/lyndsysimon>`_
@@ -31,3 +31,5 @@ Contributors
 - Matthew Keitelman `@zamattiac <https://github.com/zamattiac>`_
 - John Tordoff `@Johnetordoff <https://github.com/Johnetordoff>`_
 - Addison Schiller `@AddisonSchiller <https://github.com/AddisonSchiller>`_
+- Longze Chen `@cslzchen https://github.com/cslzchen`_
+- Jonathon Love `@jonathon-love https://github.com/jonathon-love`_

CHANGELOG

@@ -2,6 +2,24 @@
 ChangeLog
 *********
 
+0.24.0 (2018-02-01)
+===================
+- Feature: MFR now renders tiff files containing multiple images!  The file will be exported to a
+multi-page PDF, with one image per page.  (thanks, @AddisonSchiller!)
+- Feature: Matlab data files (.mat) are now rendered using the tabular formatter.  (thanks,
+@AddisonSchiller!)
+- Feature: The 3D-object renderer now supports STEP (.step and .stp) files.  (thanks,
+@AddisonSchiller!)
+- Fix: Don't send invalid payloads to MFR's metrics-tracking service.  (thanks, @AddisonSchiller!)
+- Fix: Don't reload the entire page when clicking on a tab header in the tabular renderer.
+- Code: Upgrade Pillow dependency.  (thanks, @AddisonSchiller!)
+- Code: Upgrade MFR's pym.js version to latest.  (thanks, @johnetordoff!)
+- Code: Upgrade Codepygments dependency to get the newest code highlighters.  (thanks,
+@AddisonSchiller!)
+- Code: Support rendering videos, PDFs, PDBs, and 3D objects in a local development environment.
+(thanks, @TomBaxter!)
+- Docs: Add a guide for using MFR with the OSF via docker-compose.  (thanks, @AddisonSchiller!)
+
 0.23.1 (2018-01-25)
 ===================
 - Fix: Update the Jamovi renderer to handle images with spaces in their name.  (thanks,

Dockerfile

@@ -19,9 +19,12 @@ RUN usermod -d /home www-data \
         libfreetype6-dev \
         libjpeg-dev \
         libpng12-dev \
+        libtiff5-dev \
         libxml2-dev \
         libxslt1-dev \
         zlib1g-dev \
+        # convert .step to jsc3d-compatible format
+        freecad \
         # unoconv dependencies
         unoconv \
         # pspp dependencies

README-docker-compose.md

@@ -0,0 +1,167 @@
+# MFR docker-compose README
+Note: These instructions are for running with the [Open Science Framework](https://github.com/CenterForOpenScience/osf.io/) (OSF) and its docker-compose setup.  [Check here](https://github.com/CenterForOpenScience/osf.io/blob/develop/README-docker-compose.md) for more info on working with docker/docker-compose/docker-sync. 
+
+## Adding new extensions
+When adding a new extension you will need to add it properly in `setup.py` and then run a `invoke install`.
+
+* In `setup.py` add your new extension in the form of:
+```python
+'<extension-name> = mfr.extensions.<extension-name>:<renderer-or-exporter-name>',
+```
+
+* Make sure you place it under the correct `mfr.exporters` or `mfr.renderers` section.
+
+* Check that your docker-sync is running.
+
+* If you are not in your MFR `virtualenv`, bring it up.
+
+* Run:
+
+```bash
+invoke install
+```
+
+* This will build your `mfr.egg-info` folder and add the new entrypoints.
+
+## Building a new image
+When working on an exporter or renderer, you may need a third party library to export certain file types. In order to do this easily you will need to edit the `Dockerfile` and rebuild your image.
+
+Example Dockerfile changes
+```bash
+RUN apt-get update \
+    # mfr dependencies
+    && apt-get install -y \
+        git \
+        make \
+        gcc \
+        build-essential \
+        gfortran \
+        r-base \
+        <new program> \
+```
+
+* You can usually just add the required tool to this list in the dockerfile as above.
+
+* In the same directory as your dockerfile run:
+
+``` bash
+docker build -t <image-name> ./>
+```
+* If docker hasn't cached this command recently it will take quite a while to finish.
+
+* In order to use your new image, you will have to change the `docker-compose.yml` file in your copy of the OSF. Comment out the default and add your own.
+
+```bash
+  mfr:
+    image: <image-name> # quay.io/centerforopenscience/mfr:develop
+    command: invoke server
+    restart: unless-stopped
+    ports:
+      - 7778:7778
+    env_file:
+      - .docker-compose.mfr.env
+    volumes:
+      - mfr_requirements_vol:/usr/local/lib/python3.5
+      - mfr_requirements_local_bin_vol:/usr/local/bin
+    stdin_open: true
+```
+
+* Running `docker-compose up -d` should now bring up MFR based on your new image.
+
+
+## Running mfr_requirements
+When you add a new python library to the requirements.txt, you can install it by running the `mfr_requirements` container.
+
+* Make sure you are in your OSF directory.
+
+* Bring up your MFR container with:
+
+```bash
+docker-compose up --no-deps --force-recreate mfr
+```
+
+* Then run:
+
+```bash
+docker-compose up mfr_requirements
+```
+
+* Sometimes the requirements will get stuck. Restarting the container, docker, or trying again can help.
+
+
+## Manually installing packages and programs in a docker container
+While not always the best solution, sometimes it can be useful to forgo running mfr_requirements or building an image. When this is the case, you can exec into a docker container directly and install them. This is mostly useful for testing/updating packages.
+
+* Find your container ID:
+
+```bash
+docker ps | grep mfr
+```
+* Usually you only need the first 3 characters of the id.
+
+* Exec into the container:
+
+```bash
+docker-exec -ti <id> bash
+```
+
+* Once inside the container run your command. If you are using pip, just pip install your required package. If you need to `apt-get install` something, run `apt-get update` and then you can install the package you need.
+
+## Logging information
+You can create a logger and use it to report whatever information you want.
+
+* In your python file import logging and create a logger.
+
+```python
+  import logging
+  logger = logging.getLogger(__name__)
+```
+
+* You can now log with the command `logger.info`.
+
+```python
+name = 'log me!'
+logger.info(name)
+```
+
+* To find the log, use `docker-compose logs -f --tail 1000 mfr` or recreate the container and leave it attached to your terminal. IE `docker-compose up --no-deps --no-recreate mfr`.
+
+* Make sure you remove your debugging statements before committing.
+
+## Docker is not syncing correctly
+If your docker is not syncing correctly, most of the time it can be one of a few problems.
+
+* Make sure your `docker-sync.yml` and `docker-compose.override.yml` files are set up correctly.
+
+* Run `docker-sync clean`
+
+## Setting up Keen logging in MFR or Waterbutler.
+
+* Go to https://keen.io/ and make an account.
+
+* Locate your unique logging key and ID after making a project.
+
+* Add your keys [here](https://github.com/CenterForOpenScience/waterbutler/blob/develop/waterbutler/settings.py). It should look something like this:
+
+```python
+KEEN_PRIVATE_PROJECT_ID = keen_private_config.get_nullable('PROJECT_ID', <YOUR_ID_HERE>)
+KEEN_PRIVATE_WRITE_KEY = keen_private_config.get_nullable('WRITE_KEY', <YOUR_KEY_HERE>)
+```
+
+* Restart MFR and download a file. Information should appear on your Keen stream.
+
+## Setting up Sentry logging in MFR or Waterbutler.
+
+* Go to https://sentry.io/welcome/ and create an account.
+
+* Make a python project and get your unique url.
+
+* Add your keys [here](https://github.com/CenterForOpenScience/waterbutler/blob/develop/waterbutler/settings.py).
+It should look something like this:
+
+```python
+SENTRY_DSN = config.get_nullable('SENTRY_DSN', <YOUR_SENTRY_URL_HERE>)
+```
+
+* Trigger an error in MFR or in Waterbutler and make sure it is logged correctly to Sentry. 
+

mfr/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.23.1'
+__version__ = '0.24.0'
 __import__('pkg_resources').declare_namespace(__name__)

mfr/core/remote_logging.py

@@ -120,6 +120,26 @@ async def _send_to_keen(payload, collection, project_id, write_key, action, doma
         return
 
 
+def _scrub_headers_for_keen(payload, MAX_ITERATIONS=10):
+    """ Scrub unwanted characters like \\.\\ from the keys in the keen payload """
+
+    scrubbed_payload = {}
+    for key in sorted(payload):
+        scrubbed_key = key.replace('.', '-')
+
+        # if our new scrubbed key is already in the payload, we need to increment it
+        if scrubbed_key in scrubbed_payload:
+            for i in range(1, MAX_ITERATIONS + 1):  # try MAX_ITERATION times, then give up & drop it
+                incremented_key = '{}-{}'.format(scrubbed_key, i)
+                if incremented_key not in scrubbed_payload:  # we found an unused key!
+                    scrubbed_payload[incremented_key] = payload[key]
+                    break
+        else:
+            scrubbed_payload[scrubbed_key] = payload[key]
+
+    return scrubbed_payload
+
+
 def _serialize_request(request):
     """Serialize the original request."""
     if request is None:
@@ -130,6 +150,7 @@ def _serialize_request(request):
         if k not in ('Authorization', 'Cookie', 'User-Agent',):
             headers_dict[k] = v
 
+    headers_dict = _scrub_headers_for_keen(headers_dict)
     serialized = {
         'tech': {
             'ip': request.remote_ip,

mfr/extensions/image/export.py

@@ -7,6 +7,7 @@
 
 from mfr.core import extension
 from mfr.extensions.image import exceptions
+from mfr.extensions.image.settings import EXPORT_BACKGROUND_COLOR
 
 
 class ImageExporter(extension.BaseExporter):
@@ -40,17 +41,22 @@ def export(self):
                 ratio = min(max_size[0] / image.size[0], max_size[1] / image.size[1])
                 self.metrics.add('ratio', ratio)
                 if ratio < 1:
-                    image = image.resize((round(image.size[0] * ratio), round(image.size[1] * ratio)), Image.ANTIALIAS)
-            if type in ['jpeg', 'jpg']:
-                # handle transparency
-                image = image.convert('RGBA')
-                exported_image = Image.new("RGBA", image.size, (255, 255, 255))
-                exported_image.paste(image, image)
-            else:
-                exported_image = image
-            exported_image.save(self.output_file_path, type)
-            exported_image.close()
-        except (UnicodeDecodeError, IOError) as err:
+                    image = image.resize((round(image.size[0] * ratio),
+                                          round(image.size[1] * ratio)), Image.ANTIALIAS)
+
+            # handle transparency
+            # from https://github.com/python-pillow/Pillow/issues/2609
+            if image.mode in ('RGBA', 'RGBa', 'LA') and type in ['jpeg', 'jpg']:
+                # JPEG has no transparency, so anything that was transparent gets changed to
+                # EXPORT_BACKGROUND_COLOR. Default is white.
+                background = Image.new(image.mode[:-1], image.size, EXPORT_BACKGROUND_COLOR)
+                background.paste(image, image.split()[-1])
+                image = background
+
+            image.save(self.output_file_path, type)
+            image.close()
+
+        except (UnicodeDecodeError, IOError, FileNotFoundError, OSError) as err:
             name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
             raise exceptions.PillowImageError(
                 'Unable to export the file as a {}, please check that the '

mfr/extensions/image/render.py

@@ -1,11 +1,11 @@
 import os
 
 import furl
-
 from mako.lookup import TemplateLookup
 
 from mfr.core import extension
 from mfr.extensions.image import settings
+from mfr.extensions.utils import munge_url_for_localdev
 
 
 class ImageRenderer(extension.BaseRenderer):
@@ -18,15 +18,17 @@ class ImageRenderer(extension.BaseRenderer):
     def render(self):
         self.metrics.add('needs_export', False)
         if self.metadata.ext in settings.EXPORT_EXCLUSIONS:
-            return self.TEMPLATE.render(base=self.assets_url, url=self.url)
+            download_url = munge_url_for_localdev(self.url)
+            return self.TEMPLATE.render(base=self.assets_url, url=download_url.geturl())
 
         exported_url = furl.furl(self.export_url)
         if settings.EXPORT_MAXIMUM_SIZE and settings.EXPORT_TYPE:
             exported_url.args['format'] = '{}.{}'.format(settings.EXPORT_MAXIMUM_SIZE, settings.EXPORT_TYPE)
         elif settings.EXPORT_TYPE:
             exported_url.args['format'] = settings.EXPORT_TYPE
         else:
-            return self.TEMPLATE.render(base=self.assets_url, url=self.url)
+            download_url = munge_url_for_localdev(self.url)
+            return self.TEMPLATE.render(base=self.assets_url, url=download_url.geturl())
 
         self.metrics.add('needs_export', True)
         return self.TEMPLATE.render(base=self.assets_url, url=exported_url.url)

mfr/extensions/image/settings.py

@@ -1,8 +1,8 @@
 from mfr import settings
 
-
 config = settings.child('IMAGE_EXTENSION_CONFIG')
 
 EXPORT_TYPE = config.get('EXPORT_TYPE', 'jpeg')
 EXPORT_MAXIMUM_SIZE = config.get('EXPORT_MAXIMUM_SIZE', '1200x1200')
-EXPORT_EXCLUSIONS = config.get('EXPORT_EXCLUSIONS', ['.gif', '.ico', ])
+EXPORT_EXCLUSIONS = config.get('EXPORT_EXCLUSIONS', '.gif .ico').split(' ')
+EXPORT_BACKGROUND_COLOR = config.get('EXPORT_BACKGROUND_COLOR', 'white')

mfr/extensions/jsc3d/__init__.py

@@ -1 +1,2 @@
 from .render import JSC3DRenderer  # noqa
+from .export import JSC3DExporter  # noqa