Merge branch 'release/19.0.0'

CHANGELOG

@@ -2,6 +2,19 @@
 ChangeLog
 *********
 
+19.0.0 (2019-06-19)
+===================
+- Fix: Add white background to transparent .gifs to improve zooming appearance.  The zooming code
+keeps a copy of the image behind the zoomed version to improve performance.  If the image is
+transparent the actual-size copy will be visible behind the zoomed version.  Making the background
+opaque hides the original.
+- Feature: Track when the hypothes.is sidebar is opened via Google Analytics.  (thanks, @adlius!)
+- Feature: Add timeouts to renderers that shell out to external processes for conversion.  If a
+conversion takes longer than the configured number of seconds, MFR will return an error message
+instead of waiting forever.  (thanks, @ExProbitasFiducia!)
+- Code: Remove Rackspace cache-management tasks.  MFR is now hosted on GoogleCloud, and these are no
+longer needed.  (thanks, @ExProbitasFiducia!)
+
 18.0.0 (2018-10-11)
 ===================
 - UPDATE: MFR is now following the CalVer (https://calver.org/) versioning scheme to match the

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.5-slim-jessie
+FROM python:3.5-slim-stretch
 
 # ensure unoconv can locate the uno library
 ENV PYTHONPATH=/usr/lib/python3/dist-packages
@@ -18,7 +18,7 @@ RUN usermod -d /home www-data \
         libevent-dev \
         libfreetype6-dev \
         libjpeg-dev \
-        libpng12-dev \
+        libpng-dev \
         libtiff5-dev \
         libxml2-dev \
         libxslt1-dev \
@@ -32,14 +32,13 @@ RUN usermod -d /home www-data \
         gnupg2 \
     # gosu
     && export GOSU_VERSION='1.10' \
-    && for key in \
-      # GOSU
-      B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+    && mkdir ~/.gnupg && chmod 600 ~/.gnupg && echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf \
+    && for server in hkp://ipv4.pool.sks-keyservers.net:80 \
+                     hkp://ha.pool.sks-keyservers.net:80 \
+                     hkp://pgp.mit.edu:80 \
+                     hkp://keyserver.pgp.com:80 \
     ; do \
-      gpg --keyserver hkp://ipv4.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" || \
-      gpg --keyserver hkp://keyserver.pgp.com:80 --recv-keys "$key" \
+      gpg --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || echo "Trying new server..." \
     ; done \
     && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
   	&& curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
@@ -50,29 +49,38 @@ RUN usermod -d /home www-data \
     && apt-get clean \
     && apt-get autoremove -y \
         curl \
+        gnupg2 \
     && rm -rf /var/lib/apt/lists/* \
     && pip install -U pip \
     && pip install setuptools==37.0.0 \
     && mkdir -p /code
 
-ENV LIBREOFFICE_VERSION 6.0.2.1
-ENV LIBREOFFICE_ARCHIVE LibreOffice_6.0.2.1_Linux_x86-64_deb.tar.gz
-ENV LIBREOFFICE_MIRROR_URL https://downloadarchive.documentfoundation.org/libreoffice/old/
+ENV LIBREOFFICE_VERSION 6.1.5
+ENV LIBREOFFICE_ARCHIVE LibreOffice_6.1.5_Linux_x86-64_deb.tar.gz
+ENV LIBREOFFICE_MIRROR_URL https://download.documentfoundation.org/libreoffice/stable/
 RUN apt-get update \
     && apt-get install -y \
         curl \
-    && gpg --keyserver pool.sks-keyservers.net --recv-keys AFEEAEA3 \
+        gnupg2 \
+    && for server in hkp://ipv4.pool.sks-keyservers.net:80 \
+                     hkp://ha.pool.sks-keyservers.net:80 \
+                     hkp://pgp.mit.edu:80 \
+                     hkp://keyserver.pgp.com:80 \
+    ; do \
+      gpg --keyserver "$server" --recv-keys AFEEAEA3 && break || echo "Trying new server..." \
+    ; done \
     && curl -SL "$LIBREOFFICE_MIRROR_URL/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE" -o $LIBREOFFICE_ARCHIVE \
-    && curl -SL "$LIBREOFFICE_MIRROR_URL/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE.asc" -o $LIBREOFFICE_ARCHIVE.asc \
-    && gpg --verify "$LIBREOFFICE_ARCHIVE.asc" \
-    && mkdir /tmp/libreoffice \
-    && tar -xvf "$LIBREOFFICE_ARCHIVE" -C /tmp/libreoffice/ --strip-components=1 \
-    && dpkg -i /tmp/libreoffice/**/*.deb \
-    && rm $LIBREOFFICE_ARCHIVE* \
-    && rm -Rf /tmp/libreoffice \
+        && curl -SL "$LIBREOFFICE_MIRROR_URL/$LIBREOFFICE_VERSION/deb/x86_64/$LIBREOFFICE_ARCHIVE.asc" -o $LIBREOFFICE_ARCHIVE.asc \
+        && gpg --verify "$LIBREOFFICE_ARCHIVE.asc" \
+        && mkdir /tmp/libreoffice \
+        && tar -xvf "$LIBREOFFICE_ARCHIVE" -C /tmp/libreoffice/ --strip-components=1 \
+        && dpkg -i /tmp/libreoffice/**/*.deb \
+        && rm $LIBREOFFICE_ARCHIVE* \
+        && rm -Rf /tmp/libreoffice \
     && apt-get clean \
     && apt-get autoremove -y \
         curl \
+        gnupg2 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip install unoconv==0.8.2

mfr/extensions/image/static/js/jquery.zoom.js

@@ -4,10 +4,11 @@
  * Original Copy: https://github.com/jackmoore/zoom/blob/1.7.20/jquery.zoom.js
  * Version: https://github.com/jackmoore/zoom/releases/tag/1.7.20
  *
- * The are two MFR customizations in this file, one for style and one for functionality
+ * The are three MFR customizations in this file, two for style and one for functionality
  *
  * 1. Updated code style according to `.eslintrc.js`
  * 2. Added `.on("mousewheel", function (e) { ... })` to enable further zoom by mouse wheel scroll
+ * 3. Set "background-color: white" for "zoomImage" to handle images with transparent background
  */
 
 (function ($) {
@@ -53,6 +54,7 @@
                 top: 0,
                 left: 0,
                 opacity: 0,
+                "background-color": "white",
                 width: img.width * magnify,
                 height: img.height * magnify,
                 border: "none",

mfr/extensions/jsc3d/export.py

@@ -1,13 +1,17 @@
-import os
 import shutil
-import subprocess
 from http import HTTPStatus
+from subprocess import (DEVNULL,
+                        check_call,
+                        TimeoutExpired,
+                        CalledProcessError)
+from os.path import basename, splitext
 from tempfile import NamedTemporaryFile
 
 from mfr.core import exceptions
 from mfr.core.extension import BaseExporter
-from mfr.extensions.jsc3d.settings import (FREECAD_BIN,
-                                           FREECAD_CONVERT_SCRIPT)
+from mfr.extensions.jsc3d.settings import (TIMEOUT,
+                                           FREECAD_BIN,
+                                           CONVERSION_SCRIPT)
 
 
 class JSC3DExporter(BaseExporter):
@@ -22,16 +26,14 @@ def export(self):
                 temp_source_file.name = self.source_file_path + '.step'
                 shutil.copy2(self.source_file_path, temp_source_file.name)
 
-                subprocess.check_call([
-                    FREECAD_BIN,
-                    FREECAD_CONVERT_SCRIPT,
-                    temp_source_file.name,
-                    self.output_file_path,
-                    # silence output from freecadcmd
-                ], stdout=subprocess.DEVNULL)
+                check_call(
+                    [FREECAD_BIN, CONVERSION_SCRIPT, temp_source_file.name, self.output_file_path],
+                    stdout=DEVNULL,  # silence output from freecadcmd
+                    timeout=TIMEOUT,
+                )
 
-        except subprocess.CalledProcessError as err:
-            name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
+        except CalledProcessError as err:
+            name, extension = splitext(basename(self.source_file_path))
             raise exceptions.SubprocessError(
                 'Unable to export the file in the requested format, please try again later.',
                 process='freecad',
@@ -42,3 +44,19 @@ def export(self):
                 extension=extension or '',
                 exporter_class='jsc3d',
             )
+
+        except TimeoutExpired as err:
+            name, extension = splitext(basename(self.source_file_path))
+            # The return code 52 is not the error code returned by the
+            # subprocess, but the error given to it by this waterbutler
+            # processs, for timing out.
+            raise exceptions.SubprocessError(
+                'JSC3D Conversion timed out.',
+                code=HTTPStatus.GATEWAY_TIMEOUT,
+                process='freecad',
+                cmd=str(err.cmd),
+                returncode=52,
+                path=str(self.source_file_path),
+                extension=extension or '',
+                exporter_class='jsc3d'
+            )

mfr/extensions/jsc3d/settings.py

@@ -3,6 +3,7 @@
 config = settings.child('JSC3D_EXTENSION_CONFIG')
 
 FREECAD_BIN = config.get('FREECAD_BIN', '/usr/bin/freecadcmd')
-FREECAD_CONVERT_SCRIPT = config.get('FREECAD_CONVERT_SCRIPT', '/code/mfr/extensions/jsc3d/freecad_converter.py')
+CONVERSION_SCRIPT = config.get('FREECAD_CONVERT_SCRIPT', '/code/mfr/extensions/jsc3d/freecad_converter.py')
+TIMEOUT = int(config.get('FREECAD_TIMEOUT', 30))  # In seconds
 EXPORT_TYPE = config.get('EXPORT_TYPE', '.stl')
 EXPORT_EXCLUSIONS = config.get('EXPORT_EXCLUSIONS', '.3ds .stl .obj .ctm').split(' ')

mfr/extensions/pdf/render.py

@@ -7,6 +7,7 @@
 from mfr.core import extension
 from mfr.extensions.pdf import settings
 from mfr.extensions.utils import munge_url_for_localdev, escape_url_for_template
+from mfr.settings import GOOGLE_ANALYTICS_TRACKING_ID
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +30,7 @@ def render(self):
         if self.metadata.ext.lower() not in settings.EXPORT_SUPPORTED:
             logger.debug('Extension not found in supported list!')
             return self.TEMPLATE.render(
+                ga_tracking_id=GOOGLE_ANALYTICS_TRACKING_ID,
                 base=self.assets_url,
                 url=escape_url_for_template(download_url.geturl()),
                 stable_id=self.metadata.stable_id,
@@ -46,6 +48,7 @@ def render(self):
 
         self.metrics.add('needs_export', True)
         return self.TEMPLATE.render(
+            ga_tracking_id=GOOGLE_ANALYTICS_TRACKING_ID,
             base=self.assets_url,
             url=escape_url_for_template(exported_url.url),
             stable_id=self.metadata.stable_id,

mfr/extensions/pdf/templates/viewer.mako

@@ -426,6 +426,12 @@ http://sourceforge.net/adobe/cmap/wiki/License/
     <script>
         window.MFR_STABLE_ID = '${stable_id}';
         window.MFR_FILE_NAME = '${file_name}';
+        window.GA_TRACKING_ID = '${ga_tracking_id}';
+        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+        })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+        ga('create', window.GA_TRACKING_ID, 'auto');
     </script>
     <script src="/static/js/mfr.child.hypothesis.js"></script>
     % endif

mfr/extensions/tabular/settings.py

@@ -37,3 +37,4 @@
 })
 
 PSPP_CONVERT_BIN = config.get('PSPP_CONVERT_BIN', '/usr/bin/pspp-convert')
+PSPP_CONVERT_TIMEOUT = int(config.get('PSPP_CONVERT_TIMEOUT', 30))  # In seconds

mfr/extensions/tabular/utilities.py

@@ -1,9 +1,14 @@
 import re
-import subprocess
+from http import HTTPStatus
+from subprocess import (check_call,
+                        TimeoutExpired,
+                        CalledProcessError)
 from tempfile import NamedTemporaryFile
 
-from mfr.core import exceptions
-from mfr.extensions.tabular import compat, settings
+from mfr.extensions.tabular import compat
+from mfr.core.exceptions import SubprocessError
+from mfr.extensions.tabular.settings import (PSPP_CONVERT_BIN,
+                                             PSPP_CONVERT_TIMEOUT)
 
 
 def header_population(headers):
@@ -48,20 +53,33 @@ def sav_to_csv(fp):
     """
     csv_file = NamedTemporaryFile(mode='w+b', suffix='.csv')
     try:
-        subprocess.check_call([
-            settings.PSPP_CONVERT_BIN,
-            fp.name,
-            csv_file.name,
-        ])
-    except subprocess.CalledProcessError as err:
-        raise exceptions.SubprocessError(
+        check_call(
+            [PSPP_CONVERT_BIN, fp.name, csv_file.name],
+            timeout=PSPP_CONVERT_TIMEOUT,
+        )
+    except CalledProcessError as err:
+        raise SubprocessError(
             'Unable to convert the SPSS file to CSV, please try again later.',
-            code=500,
+            code=HTTPStatus.INTERNAL_SERVER_ERROR,
             process='pspp',
             cmd=str(err.cmd),
             returncode=err.returncode,
             path=fp.name,
             extension='sav',
             exporter_class='tabular',
         )
+    except TimeoutExpired as err:
+        # The return code 52 is not the error code returned by the
+        # subprocess, but the error given to it by this waterbutler
+        # processs, for timing out.
+        raise SubprocessError(
+            'CSV Conversion timed out.',
+            code=HTTPStatus.GATEWAY_TIMEOUT,
+            process='pspp',
+            cmd=str(err.cmd),
+            returncode=52,
+            path=fp.name,
+            extension='sav',
+            exporter_class='tabular'
+        )
     return csv_file

mfr/extensions/unoconv/export.py

@@ -1,35 +1,37 @@
-import os
-import subprocess
+from http import HTTPStatus
+from os.path import basename, splitext
+from subprocess import run, CalledProcessError
 
-from mfr.core import extension
-from mfr.core import exceptions
+from mfr.core.extension import BaseExporter
+from mfr.core.exceptions import SubprocessError
+from mfr.extensions.unoconv.settings import (PORT,
+                                             ADDRESS,
+                                             UNOCONV_BIN,
+                                             UNOCONV_TIMEOUT)
 
-from mfr.extensions.unoconv import settings
 
-
-class UnoconvExporter(extension.BaseExporter):
+class UnoconvExporter(BaseExporter):
 
     def export(self):
         try:
-            subprocess.run([
-                settings.UNOCONV_BIN,
+            run([
+                UNOCONV_BIN,
                 '-n',
-                '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(settings.ADDRESS, settings.PORT),
+                '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(ADDRESS, PORT),
                 '-f', self.format,
                 '-o', self.output_file_path,
                 '-vvv',
                 self.source_file_path
-            ], check=True, timeout=settings.UNOCONV_TIMEOUT)
-
-        except subprocess.CalledProcessError as err:
-            name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
-            raise exceptions.SubprocessError(
+            ], check=True, timeout=UNOCONV_TIMEOUT)
+        except CalledProcessError as err:
+            name, extension = splitext(basename(self.source_file_path))
+            raise SubprocessError(
                 'Unable to export the file in the requested format, please try again later.',
                 process='unoconv',
                 cmd=str(err.cmd),
                 returncode=err.returncode,
                 path=str(self.source_file_path),
-                code=400,
+                code=HTTPStatus.BAD_REQUEST,
                 extension=extension or '',
                 exporter_class='unoconv',
             )

mfr/server/static/js/mfr.child.hypothesis.js

@@ -62,7 +62,34 @@
             window.document.head.appendChild(script);
             window.document.body.classList.add('show-hypothesis');
             hypothesisLoaded = true;
-        });
 
+            var sidePanelOpened = false;
+            // window.DEFAULT_URL should be the wb link in this format:
+            // https://<wb-domain>/v1/resources/<preprint-guid>/providers/osfstorage/<file-id>?direct=&mode=render
+            // TODO: parse and validate the WB URL before retrieving the preprints GUID
+            var wbLink = window.DEFAULT_URL;
+            var preprintGuid;
+            if (wbLink.split('/').length >= 6) {
+                preprintGuid = wbLink.split('/')[5];
+            } else {
+                preprintGuid = 'preprint-guid-unknown';
+            }
+            var sendAnalyticsIfExpanded = function (expanded) { 
+                if (expanded && !sidePanelOpened) {
+                    sidePanelOpened = expanded;
+                    ga('send', 'event', {
+                        eventCategory: 'Hypothesis',
+                        eventAction: 'Open Hypothesis Panel',
+                        //`eventLabel` is the guid of the preprint to which the file belongs
+                        eventLabel: preprintGuid,
+                    });
+                }
+            };
+            window.hypothesisConfig = function () {
+                return {
+                  "onLayoutChange": function (layout) { return sendAnalyticsIfExpanded(layout.expanded); }
+                };
+            };
+        });
     };
 })();

mfr/settings.py

@@ -170,3 +170,4 @@ def child(key):
 
 
 SENTRY_DSN = config.get_nullable('SENTRY_DSN', None)
+GOOGLE_ANALYTICS_TRACKING_ID = config.get_nullable('GOOGLE_ANALYTICS_TRACKING_ID', None)