Merge branch 'release/0.14.0'

.dockerignore

@@ -1,5 +1,3 @@
 .git
-*tests
-**/tests
 *.pyc
 **/*.pyc

CHANGELOG

@@ -2,6 +2,12 @@
 ChangeLog
 *********
 
+0.14.0 (2016-06-24)
+===================
+- Feature: The Dockerfile now sets up unoconv for you.
+- Fix: Character encodings for text files are now detected with the chardet library.  Inspired
+by a file that was both valid ISO-8859-1 and UTF-16 at the same time.
+
 0.13.0 (2016-06-17)
 ===================
 - Avoid an unnecessary lookup when MFR's OSF provider gets a WaterButler V1 url for
@@ -14,7 +20,7 @@ downloading. (thanks, @pattisdr!)
 
 0.12.2 (2016-06-13)
 ===================
-- Add a Dockerfile to simplify runninf MFR in dev environments.
+- Add a Dockerfile to simplify running MFR in dev environments.
 - Pin invoke to v0.11.1.  Our tasks.py is incompatible with v0.13.
 
 0.12.1 (2016-05-31)
@@ -24,7 +30,7 @@ downloading. (thanks, @pattisdr!)
 0.12.0 (2016-05-24)
 ===================
 - MFR now requires python-3.5!  Make sure to set the SERVER_DEBUG flag to false in your server
-config to avoid the hated "throw() takes 2 positional arguments but 4 were given" error.  
+config to avoid the hated "throw() takes 2 positional arguments but 4 were given" error.
 - Tabular files now sort numish columns numerically!  (thanks, @mfraezz!)
 - MFR correctly sets the Access-Control-Allow-Origin header when it receives a request with
 an Authorization header but no cookie. IOW it can now be used outside the OSF! (thanks,

Dockerfile

@@ -42,6 +42,9 @@ RUN apt-get update \
         curl \
     && rm -rf /var/lib/apt/lists/*
 
+# ensure unoconv can locate the uno library
+ENV PYTHONPATH=/usr/lib/python3/dist-packages
+
 RUN mkdir -p /code
 WORKDIR /code
 
@@ -54,6 +57,8 @@ RUN pip install --no-cache-dir -r /code/requirements.txt
 # Copy the rest of the code over
 COPY ./ /code/
 
+RUN python setup.py develop
+
 EXPOSE 7778
 
 CMD ["gosu", "nobody", "invoke", "server"]

dev-requirements.txt

@@ -1,11 +1,14 @@
+-r requirements.txt
+
+-e git+https://github.com/centerforopenscience/aiohttpretty.git@0.0.2#egg=aiohttpretty
+colorlog==2.5.0
 flake8==2.3.0
-pep8
+ipdb
 mccabe
+pep8
 pyflakes
-ipdb
 pytest==2.8.2
 pytest-cov==2.2.0
 pyzmq==14.4.1
-colorlog==2.5.0
--e git+https://github.com/centerforopenscience/aiohttpretty.git@0.0.2#egg=aiohttpretty
--r requirements.txt
+
+

docs/conf.py

@@ -173,80 +173,17 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'mfrdoc'
 
-
-# -- Options for LaTeX output --------------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #'preamble': '',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ('index', 'mfr.tex', u'mfr Documentation',
-    u'Center for Open Science', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
-
-
-# -- Options for manual page output --------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'mfr', u'mfr Documentation',
-     [u'Center for Open Science'], 1)
-]
-
-# If true, show URL addresses after external links.
-#man_show_urls = False
-
-
-# -- Options for Texinfo output ------------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    ('index', 'mfr', u'mfr Documentation',
-    u'Center for Open Science', 'mfr', 'HTML file renderer for Python',
-    ''),
+    ('documentation', False),
 ]
 
-# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
-
-# If false, no module index is generated.
-#texinfo_domain_indices = True
 
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# On RTD we can't import sphinx_rtd_theme, but it will be applied by
+# default anyway.  This block will use the same theme when building locally
+# as on RTD.
+if not on_rtd:
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

mfr/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.13.0'
+__version__ = '0.14.0'
 __import__('pkg_resources').declare_namespace(__name__)

mfr/extensions/codepygments/render.py

@@ -1,5 +1,6 @@
 import os
 
+import chardet
 import pygments.lexers
 import pygments.lexers.special
 import pygments.formatters
@@ -34,20 +35,27 @@ def cache_result(self):
     def _render_html(self, fp, ext, *args, **kwargs):
         """Generate an html representation of the file
         :param fp: File pointer
+        :param ext: File name extension
         :return: Content html
         """
         formatter = pygments.formatters.HtmlFormatter()
         data = fp.read()
 
         content, exception = None, None
-        for encoding in ['utf-8', 'utf-16', 'windows-1252', 'ISO-8859-2']:
+        try:
+            content = data.decode('utf-8')
+        except UnicodeDecodeError as e:
+            exception = e
+
+        if exception is not None:
+            encoding = chardet.detect(data)
             try:
-                content = data.decode(encoding)
-                break
+                content = data.decode(encoding['encoding'])
             except UnicodeDecodeError as e:
                 exception = e
+
         if content is None:
-            assert exception is not None, 'Got not content or exception'
+            assert exception is not None, 'Got no content or exception'
             raise exception
 
         try:

requirements.txt

@@ -1,4 +1,5 @@
 aiohttp==0.18.4
+chardet==2.3.0
 furl==0.4.2
 invoke==0.11.1
 mako==1.0.1

setup.cfg

@@ -8,4 +8,4 @@
 [flake8]
 ignore = E501,E127,E128,E265,E301,E302,F403,E731
 max-line-length = 100
-exclude = .ropeproject,tests/*
+exclude = .ropeproject,tests/*,src/*

setup.py

@@ -27,7 +27,7 @@ def parse_requirements(requirements):
         'Natural Language :: English',
         'Intended Audience :: Developers',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
     ],

tests/extensions/codepygments/files/encodings/complex.utf8.txt

@@ -0,0 +1,2 @@
+Héllö!
+แป็ด