diff --git a/.gitignore b/.gitignore index e230beecb..8552b530a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,7 @@ Thumbs.db # Python ####################### *.py[cod] -*.so -*.py[cod] +**/*.pyc # C extensions *.so diff --git a/.travis.yml b/.travis.yml index 47de7ebea..1eb6ea082 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,8 @@ language: python sudo: false python: - - "2.6" - - "2.7" - "3.3" - "3.4" - # TODO(asmacdo) - # pypy is unable to install numpy and pandas. The test should either not - # install pandas or it should install from the pypy repo. - # - "pypy" install: - travis_retry pip install -r dev-requirements.txt diff --git a/AUTHORS.rst b/AUTHORS.rst index b975b2184..ffe6bfc94 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -21,4 +21,7 @@ Contributors - Lyndsy Simon `@lyndsysimon `_ - Jeffrey Spies `@JeffSpies `_ - Matt Frazier `@mfraezz `_ -- Casey Rollins `@caseyrollins `_ \ No newline at end of file +- Casey Rollins `@caseyrollins `_ +- Michael Haselton `@icereval `_ +- Megan Kelly `@megankelly `_ +- Chris Seto `@chrisseto `_ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ed2f3c3e7..69e23dce1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -34,15 +34,14 @@ Install the development dependencies. :: - $ pip install -r dev-requirements.txt - - + $ pip install -U -r dev-requirements.txt Lastly, install mfr in development mode. :: $ python setup.py develop + $ invoke server Running tests ------------- @@ -59,9 +58,9 @@ You can also use pytest directly. :: Writing tests ------------- -Unit tests should be written for all rendering, exporting, and detection code. +Unit tests should be written for all rendering code. -Tests can be written as functions, like so: +Tests should be encapsulated within a class and written as functions, like so: .. code-block:: python @@ -69,6 +68,7 @@ Tests can be written as functions, like so: from mfr_something import render + def test_render_html(): with open('testfile.mp4') as fp: assert render.render_html(fp) == '

rendered testfile.mp4

' @@ -88,34 +88,6 @@ The above test can be rewritten like so: .. _pytest fixtures: https://pytest.org/latest/fixture.html -Using the player ----------------- - -The mfr comes with a Flask app for previewing rendered files. Create a ``files/`` subdirectory in ``player`` and copy the files you want to render into it. Then run the app from the ``player`` directory with :: - - $ invoke player - -Then browse to ``localhost:5000`` in your browser. - -Configuring the player -++++++++++++++++++++++ - -You will likely want to add additional filehandler modules to the player. The first time you run ``invoke player``, a file is created at ``player/mfr_config_local.py``. You can add additional handlers to the ``HANDLERS`` list and add any additional configuration here. - -.. code-block:: python - - # in player/mfr_config_local.py - - import mfr_image - import mfr_code_pygments - - # Add additional handlers here - HANDLERS = [ - mfr_image.Handler, - mfr_code_pygments.Handler, - ] - - Writing A File Format Package ----------------------------- @@ -126,15 +98,14 @@ There are two main pieces of a file format package are - Your :class:`FileHandler ` -Rendering/Exporting Code +Rendering Code ++++++++++++++++++++++++ -Renderers are simply callables (functions or methods) that take a file as their first argument and return a :class:`RenderResult ` which contains content(a string of the rendered HTML) and assets (a dictionary that points to lists of javascript or css sources). +Renderers are simply callables (functions or methods) that take a file as their first argument and return Here is a very simple example of function that takes a filepointer and outputs a render result with an HTML image tag. .. code-block:: python - from mfr import RenderResult def render_img_tag(filepointer): filename = filepointer.name @@ -205,51 +176,38 @@ Each package has its own directory. At a minimum, your package should include: Apart from those files, you are free to organize your rendering and export code however you want. -A typical directory structure might look like this: +A typical extension plugin directory structure might look like this: :: - mfr - └──mfr-something - ├── export-requirements.txt - ├── render-requirements.txt - ├── __init__.py - ├── render.py - ├── export.py - ├── static - │ ├── js - │ └── css - ├── tests - │ ├── __init__.py - │ └── test_something.py - ├── templates - │ └── something.html - ├── libs - │ ├── __init__.py - │ └── something_tools.py - ├── setup.py - ├── README.rst - └── configuration.py - -where "something" is a file format, e.g. "mfr_image", "mfr_movie". - -.. note:: - - You may decide to make subdirectories for rendering and exporting code if single files start to become very large. - - -Use a template -++++++++++++++ - -The fastest way to get started on a module is to use `cookiecutter template`_ for mfr modules. This will create the directory structure above. - -:: - - $ pip install cookiecutter - $ cookiecutter https://github.com/CenterForOpenScience/cookiecutter-mfr.git - -.. _cookiecutter template: https://github.com/CenterForOpenScience/cookiecutter-mfr - + modular-file-renderer + ├──mfr + │ ├── __init__.py + │ └──extensions + │ ├── __init__.py + │ └──custom-plugin + │ ├── __init__.py + │ ├── render.py + │ ├── export.py + │ ├── settings.py + │ ├── static + │ │ ├── css + │ │ └── js + │ ├── templates + │ │ └── viewer.mako + │ └── libs + │ ├── __init__.py + │ └── tools.py + ├──tests + │ ├── __init__.py + │ └──extnesions + │ ├── __init__.py + │ └──custom-plugin + │ ├── __init__.py + │ └── test_custom_plugin.py + ├── setup.py + ├── README.md + └── requirements.py Documentation diff --git a/HISTORY.rst b/HISTORY.rst deleted file mode 100644 index 7af9f7a49..000000000 --- a/HISTORY.rst +++ /dev/null @@ -1,7 +0,0 @@ -Changelog ---------- - -0.1.0 (unreleased) -++++++++++++++++++ - -* First release. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 60d6b53ae..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include *.rst LICENSE \ No newline at end of file diff --git a/NOTICE b/NOTICE index de82d67aa..2fe6270b2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,38 +1,2 @@ mfr includes code from 3rd-party libraries, whose licenses are listed below. -Flask License -============= - -Copyright (c) 2014 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..05a659b25 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +#MFR + +**WARNING: mfr is in a very alpha stage of development. As such, it's API is in constant flux. Expect many breaking changes.** + +**mfr** (short for "modular file renderer") is a Python package for rendering files to HTML. + +### startup commands + +```bash +# Make sure that you are using >= python3.3 +pip install -U -r requirements.txt +python setup.py develop +invoke server +``` + +### Create your own module + +Interested in adding support for a new provider or file format? Check out the CONTRIBUTING.rst docs. + +### License + +Copyright 2013-2015 Center for Open Science + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index 248a3b427..000000000 --- a/README.rst +++ /dev/null @@ -1,61 +0,0 @@ -*** -mfr -*** -.. image:: https://travis-ci.org/CenterForOpenScience/modular-file-renderer.svg?branch=dev - :target: https://travis-ci.org/CenterForOpenScience/modular-file-renderer - -**WARNING: mfr is in a very alpha stage of development. As such, it's API is in constant flux. Expect many breaking changes.** - -**mfr** (short for "modular file renderer") is a Python package for rendering files to HTML. - -.. code-block:: python - - import mfr - import mfr_image - - # Enable the mfr_image module - mfr.register_filehandler(mfr_image.Handler) - - with open('hello.jpg') as filepointer: - mfr.render(filepointer, alt="Hello world") - # => 'Hello world' - - -Requirements -============ - -- Python >= 2.6 or >= 3.3 - - -Available modules -================= - -There are a number of 3rd-party modules available. - -- `mfr_md `_ - -Make your own, then submit a pull request to add it to this list! - - -Create your own module -====================== - -Interested in adding support for a new file format? Check out the CONTRIBUTING.rst docs. - - -License -======= - -Copyright 2013-2015 Center for Open Science - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/conftest.py b/conftest.py deleted file mode 100644 index e6fc73ee4..000000000 --- a/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Project-wide test configuration, including fixutres that can be -used by any module. - -Example test: :: - - def test_my_renderer(fakefile): - assert my_renderer(fakefile) == '..expected result..' - -""" -import pytest -import mock - -@pytest.fixture -def fakefile(): - """A simple file-like object.""" - mockfile = mock.Mock(spec=open) - mockfile.return_value = '' - mockfile.name = 'fakefile.md' - return mockfile diff --git a/dev-requirements.txt b/dev-requirements.txt index b5f974080..268fb0099 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,16 +1,31 @@ # Distribution -wheel +# wheel +# colorlog==2.5.0 # Task runner -invoke +# invoke # Docs -sphinx +# sphinx + +# Server +# aiohttp==0.14.1 +# tornado==4.0.2 +# stevedore==1.2.0 +# git+https://github.com/jmcarp/raven-python +# celery==3.1.17 # Testing -mock -pytest -tox>=1.5.0 +# mock +# pytest +# tox>=1.5.0 -# Previewer dependencies -Flask +decorator==3.4.0 +flake8==2.3.0 +ipdb==0.8 +pytest==2.6.4 +pytest-cov==1.8.1 +pyzmq==14.4.1 +colorlog==2.5.0 +-e git+https://github.com/centerforopenscience/aiohttpretty.git#egg=aiohttpretty +-r requirements.txt diff --git a/docs/api_reference.rst b/docs/api_reference.rst index 4678c1ef2..c6316052c 100644 --- a/docs/api_reference.rst +++ b/docs/api_reference.rst @@ -77,12 +77,12 @@ mfr_ipynb :inherited-members: -.. currentmodule:: mfr_movie +.. currentmodule:: mfr_video -mfr_movie +mfr_video --------- -.. automodule:: mfr_movie +.. automodule:: mfr_video :members: :inherited-members: diff --git a/docs/conf.py b/docs/conf.py index b061c7448..86625245e 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,7 +17,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) -import mfr +import mfr # noqa sys.path.append(os.path.abspath("_themes")) # -- General configuration ----------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index 51958a12e..1e14b379f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,18 +10,6 @@ Release v\ |version|. (:ref:`Installation `) **mfr** (short for "modular file renderer") is a Python package for rendering files to HTML. -.. code-block:: python - - import mfr - import mfr_image - - # Enable the ImageModule - mfr.register_filehandler(mfr_image.Handler) - - with open('hello.jpg') as filepointer: - mfr.render(filepointer, alt="Hello world").content - # => 'Hello world' - Ready to dive in? ----------------- diff --git a/docs/install.rst b/docs/install.rst index 24b1e71bf..8ada938f1 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -14,27 +14,13 @@ Or download one of the following: * tarball_ * zipball_ -Once you have the source, you can install it into your site-packages with :: +To install the base requirements, run: - $ python setup.py install - -This will install mfr and its core modules. Each module may have its own requirements and they be installed using the CLI:: - - # install all plugin requirements - mfr install all + $ pip install -r requirements.txt - # install requirements for a specific core module - mfr install mfr_code_pygments +Once you have the source and requirements, you can install it into your site-packages with :: -Additionally, the ``[-e | -r]`` flags will install only exporter or render requirements, respectively:: - - # install all render requirements - mfr -r install all - -You can alternatively install the requirements by passing the -requirements.txt file to pip:: - - # install render-requirements for specific core module - pip install -r /path/to/mfr_code_pygments/render-requirements.txt + $ python setup.py install .. _Github: https://github.com/CenterForOpenScience/modular-file-renderer .. _tarball: https://github.com/CenterForOpenScience/modular-file-renderer/tarball/master diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a8f479e27..a7665c2cd 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,20 +4,6 @@ Quickstart ********** -Before we start rendering, we need to enable a file format's module. To do this, use :func:`mfr.register_filehandler `, passing in a :class:`FileHandler `. - -Let's use the ``mfr_code_pygments`` module as an example. - -.. code-block:: python - - import mfr - import mfr_code_pygments - - # Enable the module's Handler - mfr.register_filehandler(mfr_code_pygments.Handler) - - -Then call :func:`mfr.detect ` with a file object, which returns an instance of a handler that can handle the file, or ``None`` if no valid handler is registered. .. code-block:: python @@ -97,30 +83,7 @@ Configuration is stored on a ``mfr.config``, which can be modified like a dictio Using Static Files ================== -Many renderers require static files (e.g. CSS and Javascript). To retrieve the static files for a file handler, call its :meth:`get_assets ` method. This will return a dictionary which maps file extensions to a list of paths. - -.. code-block:: python - - import mfr - import mfr_code_pygments - - mfr.config['STATIC_URL'] = '/static' - handler = mfr_code_pygments.Handler() - handler.get_assets()['css'] - # ['/static/mfr_code_pygments/css/autumn.css', - # '/static/mfr_code_pygments/css/borland.css', ... - -Copying Static Assets ---------------------- - -To copy all necessary static assets to your app's static folder, use :func:`collect_static `. - -.. code-block:: python - - # Static assets will be copied here - mfr.config['STATIC_FOLDER'] = '/app/static' - mfr.collect_static() # Copies static files to STATIC_FOLDER - +Many renderers require static files (e.g. CSS and Javascript). To retrieve the static files for a file renderer, the object has a 'assets_url' that serves as the base path. Next Steps ========== diff --git a/docs/settingup.rst b/docs/settingup.rst new file mode 100644 index 000000000..ec1e5a691 --- /dev/null +++ b/docs/settingup.rst @@ -0,0 +1,34 @@ +Setting Up +========== + +Make sure that you are using >= python3.3 + +Install requirements + +.. code-block:: bash + + pip install -U -r requirements.txt + +Or for some nicities + +.. code-block:: bash + + pip install -U -r dev-requirements.txt + +Required by the stevedore module. Allows for dynamic importing of providers + +.. code-block:: bash + + python setup.py develop + +Start the server + +.. note + + The server is extremely tenacious thanks to stevedore and tornado + Syntax errors in the :mod:`mfr.providers` will not crash the server + In debug mode the server will automatically reload + +.. code-block:: bash + + invoke server diff --git a/mfr/__init__.py b/mfr/__init__.py index 85f645b0f..2701d57a3 100755 --- a/mfr/__init__.py +++ b/mfr/__init__.py @@ -1,29 +1,2 @@ -"""The mfr core module.""" -# -*- coding: utf-8 -*- -import os - -__version__ = '0.1.0-alpha' -__author__ = 'Center for Open Science' - - -from mfr.core import ( - render, - detect, - FileHandler, - get_file_extension, - register_filehandler, - register_filehandlers, - export, - get_file_exporters, - get_namespace, - get_registry, - config, - collect_static, - RenderResult, -) - -from mfr._config import Config - - -PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__)) -# flake8: noqa \ No newline at end of file +__version__ = '0.1.0' +__import__('pkg_resources').declare_namespace(__name__) diff --git a/mfr/_config.py b/mfr/_config.py deleted file mode 100644 index 22df87bfa..000000000 --- a/mfr/_config.py +++ /dev/null @@ -1,158 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Config module from flask.config - -:copyright: (c) 2011 by Armin Ronacher. -:license: BSD, see NOTICE for more details. -""" - -import imp -import os -import errno - - -class ConfigAttribute(object): - """Makes an attribute forward to the config""" - - def __init__(self, name, get_converter=None): - self.__name__ = name - self.get_converter = get_converter - - def __get__(self, obj, type=None): - if obj is None: - return self - rv = obj.config[self.__name__] - if self.get_converter is not None: - rv = self.get_converter(rv) - return rv - - def __set__(self, obj, value): - obj.config[self.__name__] = value - - -class Config(dict): - """Works exactly like a dict but provides ways to fill it from files - or special dictionaries. There are two common patterns to populate the - config. - - Either you can fill the config from a config file:: - - app.config.from_pyfile('yourconfig.cfg') - - Or alternatively you can define the configuration options in the - module that calls :meth:`from_object` or provide an import path to - a module that should be loaded. It is also possible to tell it to - use the same module and with that provide the configuration values - just before the call:: - - DEBUG = True - SECRET_KEY = 'development key' - app.config.from_object(__name__) - - In both cases (loading from any Python file or loading from modules), - only uppercase keys are added to the config. This makes it possible to use - lowercase values in the config file for temporary values that are not added - to the config or to define the config keys in the same file that implements - the application. - - Probably the most interesting way to load configurations is from an - environment variable pointing to a file:: - - app.config.from_envvar('YOURAPPLICATION_SETTINGS') - - In this case before launching the application you have to set this - environment variable to the file you want to use. On Linux and OS X - use the export statement:: - - export YOURAPPLICATION_SETTINGS='/path/to/config/file' - - On windows use `set` instead. - - :param defaults: an optional dictionary of default values - :param root_path: path to which files are read relative from. - """ - - def __init__(self, defaults=None, root_path='.'): - dict.__init__(self, defaults or {}) - self.root_path = root_path - - def from_envvar(self, variable_name, silent=False): - """Loads a configuration from an environment variable pointing to - a configuration file. This is basically just a shortcut with nicer - error messages for this line of code:: - - app.config.from_pyfile(os.environ['YOURAPPLICATION_SETTINGS']) - - :param variable_name: name of the environment variable - :param silent: set to `True` if you want silent failure for missing - files. - :return: bool. `True` if able to load config, `False` otherwise. - """ - rv = os.environ.get(variable_name) - if not rv: - if silent: - return False - raise RuntimeError('The environment variable %r is not set ' - 'and as such configuration could not be ' - 'loaded. Set this variable and make it ' - 'point to a configuration file' % - variable_name) - return self.from_pyfile(rv, silent=silent) - - def from_pyfile(self, filename, silent=False): - """Updates the values in the config from a Python file. This function - behaves as if the file was imported as module with the - :meth:`from_object` function. - - :param filename: the filename of the config. This can either be an - absolute filename or a filename relative to the - root path. - :param silent: set to `True` if you want silent failure for missing - files. - - .. versionadded:: 0.7 - `silent` parameter. - """ - filename = os.path.join(self.root_path, filename) - d = imp.new_module('config') - d.__file__ = filename - try: - with open(filename) as config_file: - exec(compile(config_file.read(), filename, 'exec'), d.__dict__) - except IOError as e: - if silent and e.errno in (errno.ENOENT, errno.EISDIR): - return False - e.strerror = 'Unable to load configuration file (%s)' % e.strerror - raise - self.from_object(d) - return True - - def from_object(self, obj): - """Updates the values from the given object. An object can be of one - of the following two types: - - - a string: in this case the object with that name will be imported - - an actual object reference: that object is used directly - - Objects are usually either modules or classes. - - Just the uppercase variables in that object are stored in the config. - Example usage:: - - app.config.from_object('yourapplication.default_config') - from yourapplication import default_config - app.config.from_object(default_config) - - You should not use this function to load the actual configuration but - rather configuration defaults. The actual config should be loaded - with :meth:`from_pyfile` and ideally from a location not within the - package because the package might be installed system wide. - - :param obj: an import name or object - """ - for key in dir(obj): - if key.isupper(): - self[key] = getattr(obj, key) - - def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self)) diff --git a/mfr/cli.py b/mfr/cli.py deleted file mode 100755 index 8996d7b53..000000000 --- a/mfr/cli.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -"""Command Line tool for the modular file renderer""" - -import os -import pip -import sys -import argparse - -HERE = os.path.dirname(os.path.abspath(__file__)) -EXT_PATH = os.path.join(HERE, 'ext') -WHEELHOUSE_PATH = os.environ.get('WHEELHOUSE') - - -def pip_install(path, filename): - """Use pip to install from a requirements file - - :param path: location of file - :param filename: name of requirements file - """ - file_location = (os.path.join(path, filename)) - - if os.path.isfile(file_location): - pip_args = ['install', '-r', file_location] - if WHEELHOUSE_PATH: - pip_args.extend(['--use-wheel', '--find-links', WHEELHOUSE_PATH]) - pip.main(pip_args) - - -def plugin_requirements(render, export, plugins): - """Install the requirements of the core plugins - - :param render: install only render requirements - :param export: install only export requirements - :param plugins: list of plugins to install requirements of - """ - - if plugins == ['all']: - path_list = [ - os.path.join(EXT_PATH, directory) - for directory in os.listdir(EXT_PATH) - if os.path.isdir(os.path.join(EXT_PATH, directory)) - ] - else: - path_list = [os.path.join(EXT_PATH, plugin) for plugin in plugins] - - for path in path_list: - if os.path.isdir(path): - if not export: - pip_install(path, 'render-requirements.txt') - if not render: - pip_install(path, 'export-requirements.txt') - else: - print('Plugin with name "{plugin}" not found. Skipping...'.format( - plugin=os.path.basename(os.path.normpath(path)) - )) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('command') - parser.add_argument('-e', '--export', help='Install only export requirements', - action='store_true') - parser.add_argument('-r', '--render', help='Install only render requirements', - action='store_true') - parser.add_argument('plugin', nargs='*', help='List of plugins to install reqs') - - args = parser.parse_args() - if args.command == 'install': - if not args.plugin: - print('Must provide at least one plugin name to install') - sys.exit(1) - plugin_requirements(args.render, args.export, args.plugin) - else: - print('Invalid subcommand: "{command}"'.format(command=args.command)) - sys.exit(1) - -if __name__ == "__main__": - - main() diff --git a/mfr/compat.py b/mfr/compat.py deleted file mode 100644 index 3e5dcce41..000000000 --- a/mfr/compat.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import division -import sys - -PY2 = int(sys.version[0]) == 2 - -if PY2: - range = xrange - string_types = (str, unicode) - unicode = unicode - basestring = basestring -else: - range = range - string_types = (str,) - unicode = str - basestring = (str, bytes) diff --git a/mfr/core.py b/mfr/core.py deleted file mode 100644 index e154ad880..000000000 --- a/mfr/core.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- -"""Core functions and classes. - -Basic Usage: :: - - from mfr.core import detect - - with open('myfile.jpg', 'r') as fp: - handler = detect(fp) - if handler: - html = handler.render(fp) -""" -import os -import shutil -import inspect -import logging -from collections import defaultdict - -from mfr._config import Config -from mfr.exceptions import ConfigurationError, MFRError - -logger = logging.getLogger(__name__) - -_defaults = { - 'INCLUDE_STATIC': False, - 'EXCLUDE_LIBS': [] -} -#: Global mfr configuration object -config = Config(defaults=_defaults) - -config['HANDLERS'] = [] - - -def register_filehandler(file_handler): - """Register a new file handler. - Usage: :: - - register_filehandler(mfr_movie.Handler) - - :param FileHandler file_handler: A FileHandler class. - """ - reg = get_registry() - if file_handler not in reg: - get_registry().append(file_handler) - return True - return False - - -def register_filehandlers(handlers): - """Register multiple file handlers. - Usage: :: - - register_file_handlers([ - mfr_image.Handler, - mfr_movie.Handler, - mfr_tabular.Handler]) - - :param list handlers: A list of FileHandler classes. - """ - for each in handlers: - register_filehandler(each) - - -def get_registry(): - """Get the current list of registered filehandlers. - - :rtype: list - """ - return config["HANDLERS"] - - -def clear_registry(): - """Reset the list of registered handlers.""" - config["HANDLERS"] = [] - - -def reset_config(): - """Reset config defaults and empty the registry of FileHandlers.""" - global config - config.clear() - config.update(_defaults) - config["HANDLERS"] = [] - - -def detect(fp, handlers=None, instance=True, many=False, *args, **kwargs): - """Return a :class:`FileHandler ` for a given file, - or ``False`` if no handler could be found for the file. - - :param list handlers: A list of filehandler names to try. If ``None``, - try all registered handlers. - :return: A FileHandler that can handle the file, or False if no handler was - found. - """ - handlers = handlers or get_registry() - valid_handlers = [] - for HandlerClass in handlers: - handler = HandlerClass() - if handler.detect(fp, *args, **kwargs): - handler_obj = handler if instance else HandlerClass - if many: - valid_handlers.append(handler_obj) - else: - return handler_obj - if many: - return valid_handlers - else: - return None - - -def render(fp, handler=None, renderer=None, *args, **kwargs): - """Core rendering function. Return the rendered HTML for a given file. - - :param File fp: A file pointer object to render. - :param FileHandler: The file handler class to use. - :param str renderer: The name of the renderer function to use (must be a key in - in the handler class's `renderers` dictionary) - """ - # Get the specified handler, detect it if not given - handler = handler or detect(fp, many=False, instance=True) - if not handler: - #raise MFRError('No available handler that can handle {name}.' - # .format(name=os.path.basename(os.path.normpath(fp.name)))) - raise MFRError('No renderer currently available for this file type.') - return handler.render(fp, renderer=renderer, *args, **kwargs) - - -def export(fp, handler=None, exporter=None, *args, **kwargs): - """Core exporting function. Return the requested file. - - :param File fp: A file pointer object to export. - :param FileHandler: The file handler class to use. - :param str exporter: The name of the exporter function to use (must be a key in - in the handler class's `exporters` dictionary) - """ - # Get the specified handler, detect it if not given - HandlerClass = handler or detect(fp) - if not HandlerClass: - raise MFRError('No available handler with name {handler}.' - .format(handler=handler)) - - handler = HandlerClass - return handler.export(fp, exporter=exporter, *args, **kwargs) - - -def get_file_extension(path, lower=True): - """Get the file extension for a given file path.""" - ext = os.path.splitext(path)[1] - return ext.lower() if lower else ext - - -def get_file_exporters(handler): - """Get list of exporters for a given file handler.""" - try: - return handler.exporters.keys() - except AttributeError: - return None - -def assets_by_extension(assets): - """Given a list of assets, return a dictionary keyed by extension - - :param list assets: List of asset paths (strings) - :rtype: dict - """ - ret = defaultdict(list) - for asset in assets: - key = get_file_extension(asset).lstrip('.') - ret[key].append(asset) - return ret - - -class RenderResult(object): - """ An object that contains the html representation of content and any - assets that should be included. - """ - - def __init__(self, content, assets=None): - """ - Initialize a Render result. - - :param content: html representation of content - :param assets: css, javascript, or other assets to be included - """ - self.content = content - if isinstance(assets, (list, tuple)): - self.assets = assets_by_extension(assets) - elif isinstance(assets, dict): # assets is a dict - self.assets = defaultdict(list, assets) - else: # assets is None - self.assets = defaultdict(list) - - def __str__(self): - return str(self.content) - - def __repr__(self): - return ''.format(self.content) - - def __contains__(self, obj): - """Implements the ``in`` keyword.""" - return obj in str(self.content) - -class FileHandler(object): - """Abstract base class from which all file handlers must inherit. - """ - #: Maps renderer names to renderer callables, e.g. {'html': render_img_html} - renderers = {} - #: Maps exporter names to exporter callables, e.g. {'png': export_png} - exporters = {} - - default_renderer = 'html' - default_exporter = None - - def __init__(self): - self.__assets = defaultdict(list) - - def detect(self, fp): - """Return whether a given file can be handled by this file handler. - MUST be implemented by descendant classes. - """ - raise NotImplementedError('Must define detect method.') - - def render(self, fp, renderer=None, *args, **kwargs): - """Return the rendered HTML for a file. - - :param fp: A file-like object. - :param str renderer: The name of the renderer to use. If `None`, - the default_renderer is used. - """ - render_func = self.renderers.get(renderer or self.default_renderer) - # TODO(sloria): check if render_func is callable - if render_func: - return render_func(fp, *args, **kwargs) - else: - raise MFRError('`render` method called with no renderer specified and ' - 'no default.') - - def export(self, fp, exporter=None, *args, **kwargs): - """Export a file to a different format. - - :param str exporter: The name of the exporter to use. If `None`, - the default_exporter is used. - """ - export_func = self.exporters.get(exporter or self.default_exporter) - if export_func: - return export_func(fp, *args, **kwargs) - else: - raise MFRError('`export` method called with no exporter specified and ' - 'no default.') - - def iterstatic(self, url=True): - """Iterates through the static asset files for the filehandler, - yielding absolute paths to the files. - - :param bool url: If ``True``, return the static url for each asset. - If ``False``, return the static folder for each asset. - """ - static_folder = get_static_path_for_handler(self.__class__) - for root, dirs, files in os.walk(static_folder): - for filename in files: - absolute_path = os.path.join(root, filename) - if url: - namespace = get_namespace(self) - static_path = os.path.relpath(absolute_path, static_folder) - try: - yield os.path.join(config['ASSETS_URL'], namespace, static_path) - except KeyError: - raise KeyError('ASSETS_URL is not configured.') - else: - yield absolute_path - - def get_assets(self, extension=None): - """Get the urls for this handler's static assets. Return either a dict - keyed by extension or a list of assets if ``extension`` is provided. - - Usage: :: - - >>> handler.get_assets() - {'css': '/static/myformat/style.css', 'js': '/static/myformat/script.js'} - - """ - - if not self.__assets: - for asset in self.iterstatic(url=True): - ext = get_file_extension(asset).lstrip('.') - if ext: - self.__assets[ext].append(asset) - else: - self.__assets['_'].append(asset) - if extension: - return self.__assets[extension] - return self.__assets - - -def get_assets_from_list(assets_uri_base, ext, asset_list=None): - """ - Generate an list of full uri paths to each assets, excluding any files - listed in the Config's EXCLUDE_LIBS - """ - - return [ - os.path.join(assets_uri_base, ext, filepath) - for filepath in asset_list - if filepath not in config.get('EXCLUDE_LIBS') - ] - - -def _get_dir_for_class(cls): - """Return the absolute directory where a class resides.""" - fpath = inspect.getfile(cls) - return os.path.abspath(os.path.dirname(fpath)) - - -def get_static_path_for_handler(handler_cls): - """Return the absolute path for a given FileHandler class. - Defaults to a ``static`` folder within the same directory of the handler's - module. - """ - # If STATIC_PATH is defined, use that - if hasattr(handler_cls, 'STATIC_PATH'): - static_path = handler_cls.STATIC_PATH - # Otherwise assume 'static' dir is in the same directory as - # the handler class's module - else: - static_path = os.path.join(_get_dir_for_class(handler_cls), 'static') - return static_path - -def get_static_url_for_handler(handler_cls): - if hasattr(handler_cls, 'ASSETS_URL'): - url = os.path.join(config['ASSETS_URL'], handler_cls.ASSETS_URL) - else: - url = os.path.join(config['ASSETS_URL'], get_namespace(handler_cls)) - return url - - -def copy_dir(src, dest): - """Recursively copies a directory's contents.""" - try: - shutil.copytree(src, dest) - except shutil.Error as err: - logger.warn(err) - except OSError as err: - if os.path.exists(dest): - logger.debug('Skipping {dest} (already exists)'.format(dest=dest)) - else: - raise err - - -def get_namespace(handler_cls): - """Given a FileHandler class, return the namespace used by collect_static. - The namespace defines the name of the folder that a file module's static - assets will be copied to. - """ - # If 'namespace' is defined on the class, use that - if hasattr(handler_cls, 'namespace'): - return handler_cls.namespace - # Otherwise use the base name of the module - else: - # mypackage.mymodule.MyHandler => 'mymodule' - return handler_cls.__module__.split('.')[-1] - -def collect_static(dest=None, dry_run=False): - """Collect all static assets for registered handlers to a single directory. - Files will be copied to ``dest``, if specified, or the STATIC_PATH config - variable. - """ - dest_ = dest or config.get('ASSETS_FOLDER') - if not dest_: - raise ConfigurationError('ASSETS_FOLDER has not been configured.') - for handler_cls in get_registry(): - static_path = get_static_path_for_handler(handler_cls) - namespaced_destination = os.path.join(dest_, get_namespace(handler_cls)) - if dry_run: - print('Pretending to copy {static_path} to {namespaced_destination}.' - .format(**locals())) - else: - if os.path.exists(static_path): - copy_dir(static_path, namespaced_destination) diff --git a/mfr/ext/code_pygments/tests/__init__.py b/mfr/core/__init__.py similarity index 100% rename from mfr/ext/code_pygments/tests/__init__.py rename to mfr/core/__init__.py diff --git a/mfr/core/exceptions.py b/mfr/core/exceptions.py new file mode 100644 index 000000000..1b8c87ae4 --- /dev/null +++ b/mfr/core/exceptions.py @@ -0,0 +1,32 @@ +import waterbutler.core.exceptions + + +class PluginError(waterbutler.core.exceptions.PluginError): + + def as_html(self): + return ''' + + + '''.format(self.message) + + +class RendererError(PluginError): + """The MFR related errors raised + from a :class:`mfr.core.renderer` should + inherit from RendererError + """ + + +class ProviderError(PluginError): + """The MFR related errors raised + from a :class:`mfr.core.provider` should + inherit from ProviderError + """ + + +class DownloadError(ProviderError): + pass + + +class MetadataError(ProviderError): + pass diff --git a/mfr/core/extension.py b/mfr/core/extension.py new file mode 100644 index 000000000..16d0e9a9d --- /dev/null +++ b/mfr/core/extension.py @@ -0,0 +1,46 @@ +import abc + + +class BaseExporter(metaclass=abc.ABCMeta): + + def __init__(self, url, file_path, assets_url, ext): + self.url = url + self.file_path = file_path + self.assets_url = '{}/{}'.format(assets_url, self._get_module_name()) + self.extension = ext + + @abc.abstractmethod + def export(self): + pass + + def _get_module_name(self): + return self.__module__ \ + .replace('mfr.extensions.', '', 1) \ + .replace('.export', '', 1) + + +class BaseRenderer(metaclass=abc.ABCMeta): + + def __init__(self, url, download_url, file_path, assets_url, ext): + self.url = url + self.download_url = download_url + self.file_path = file_path + self.assets_url = '{}/{}'.format(assets_url, self._get_module_name()) + self.extension = ext + + @abc.abstractmethod + def render(self): + pass + + @abc.abstractproperty + def file_required(self): + pass + + @abc.abstractproperty + def cache_result(self): + pass + + def _get_module_name(self): + return self.__module__ \ + .replace('mfr.extensions.', '', 1) \ + .replace('.render', '', 1) diff --git a/mfr/core/provider.py b/mfr/core/provider.py new file mode 100644 index 000000000..eab230464 --- /dev/null +++ b/mfr/core/provider.py @@ -0,0 +1,16 @@ +import abc + + +class BaseProvider(metaclass=abc.ABCMeta): + + def __init__(self, request, url): + self.request = request + self.url = url + + @abc.abstractmethod + def metadata(self): + pass + + @abc.abstractmethod + def download(self): + pass diff --git a/mfr/core/utils.py b/mfr/core/utils.py new file mode 100644 index 000000000..2a6535f37 --- /dev/null +++ b/mfr/core/utils.py @@ -0,0 +1,91 @@ +import asyncio + +import aiohttp + +from stevedore import driver +from raven.contrib.tornado import AsyncSentryClient + +from mfr import settings +from mfr.core import exceptions + + +sentry_dns = settings.get('SENTRY_DSN', None) + + +class AioSentryClient(AsyncSentryClient): + + def send_remote(self, url, data, headers=None, callback=None): + headers = headers or {} + if not self.state.should_try(): + message = self._get_log_message(data) + self.error_logger.error(message) + return + + future = aiohttp.request('POST', url, data=data, headers=headers) + asyncio.async(future) + + +if sentry_dns: + client = AioSentryClient(sentry_dns) +else: + client = None + + +def make_provider(name, request, url): + """Returns an instance of :class:`mfr.core.provider.BaseProvider` + + :param str name: The name of the provider to instantiate. (osf) + :param dict url: + + :rtype: :class:`mfr.core.provider.BaseProvider` + """ + manager = driver.DriverManager( + namespace='mfr.providers', + name=name.lower(), + invoke_on_load=True, + invoke_args=(request, url, ), + ) + return manager.driver + + +def make_exporter(name, file_path, ext, type): + """Returns an instance of :class:`mfr.core.extension.BaseExporter` + + :param str name: The name of the extension to instantiate. (.jpg, .docx, etc) + :param dict file_path: + :param dict ext: + :param dict type: The exported file type + + :rtype: :class:`mfr.core.extension.BaseExporter` + """ + try: + return driver.DriverManager( + namespace='mfr.exporters', + name=(name and name.lower()) or 'none', + invoke_on_load=True, + invoke_args=(file_path, ext, type), + ).driver + except RuntimeError: + raise exceptions.RendererError('No exporter could be found for the file type requested.', code=400) + + +def make_renderer(name, url, download_url, file_path, assets_url, ext): + """Returns an instance of :class:`mfr.core.extension.BaseRenderer` + + :param str name: The name of the extension to instantiate. (.jpg, .docx, etc) + :param dict url: + :param dict file_path: + :param dict assets_url: + :param dict ext + + :rtype: :class:`mfr.core.extension.BaseRenderer` + """ + try: + return driver.DriverManager( + namespace='mfr.renderers', + name=(name and name.lower()) or 'none', + invoke_on_load=True, + invoke_args=(url, download_url, file_path, assets_url, ext, ), + ).driver + except RuntimeError: + raise exceptions.RendererError('No renderer could be found for the file type requested.', code=400) diff --git a/mfr/exceptions.py b/mfr/exceptions.py deleted file mode 100644 index 92367961c..000000000 --- a/mfr/exceptions.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -"""Exception classes for the mfr package.""" - - -class MFRError(Exception): - """Base exception from which all MFR-related errors inherit.""" - pass - - -class ConfigurationError(MFRError): - """Error raised when MFR is improperly configured.""" - pass - - -class RenderError(MFRError): - """Base exception for all rendering related errors""" - pass diff --git a/mfr/ext/__init__.py b/mfr/ext/__init__.py deleted file mode 100644 index 901263a7e..000000000 --- a/mfr/ext/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -from mfr.ext import ( - audio, - code_pygments, - docx, - image, - ipynb, - movie, - pdb, - pdf, - rst, - tabular, -) - -ALL_HANDLERS = [ - audio.Handler, - code_pygments.Handler, - docx.Handler, - image.Handler, - ipynb.Handler, - movie.Handler, - pdb.Handler, - pdf.Handler, - rst.Handler, - tabular.Handler, -] diff --git a/mfr/ext/audio/__init__.py b/mfr/ext/audio/__init__.py deleted file mode 100644 index 81180b4dd..000000000 --- a/mfr/ext/audio/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from mfr.core import FileHandler, get_file_extension -from .render import render_audio_tag - -EXTENSIONS = [ - '.mp3', - '.ogg', - '.wav', -] - - -class Handler(FileHandler): - """FileHandler for audio files.""" - - renderers = { - 'html': render_audio_tag, - } - - def detect(self, fp): - return get_file_extension(fp.name) in EXTENSIONS diff --git a/mfr/ext/audio/render.py b/mfr/ext/audio/render.py deleted file mode 100644 index 5998aa391..000000000 --- a/mfr/ext/audio/render.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Audio renderer module.""" -from mfr.core import RenderResult - - -def render_audio_tag(fp, src=None): - """Create a simple audio tag for a static audio file - - :param fp: File pointer. - :param src: The path to the audio file. - :return: RenderResult object containing html audio tag for given file""" - - src = src or fp.name - - content = '