diff --git a/mfr/extensions/tabular/exceptions.py b/mfr/extensions/tabular/exceptions.py index 36b6532fc..59f461eb6 100644 --- a/mfr/extensions/tabular/exceptions.py +++ b/mfr/extensions/tabular/exceptions.py @@ -29,9 +29,14 @@ class TableTooBigError(TabularRendererError): __TYPE = 'tabular_table_too_big' - def __init__(self, message, *args, code: int=400, **kwargs): + def __init__(self, message, *args, code: int=400, nbr_cols: int=0, nbr_rows: int=0, **kwargs): super().__init__(message, *args, code=code, **kwargs) - self.attr_stack.append([self.__TYPE, {}]) + self.nbr_cols = nbr_cols + self.nbr_rows = nbr_rows + self.attr_stack.append([self.__TYPE, { + 'nbr_cols': self.nbr_cols, + 'nbr_rows': self.nbr_rows + }]) class UnexpectedFormattingError(TabularRendererError): @@ -42,3 +47,20 @@ def __init__(self, message, *args, code: int=500, formatting_function: str='', * super().__init__(message, *args, code=code, **kwargs) self.formatting_function = formatting_function self.attr_stack.append([self.__TYPE, {'formatting_function': self.formatting_function}]) + + +class FileTooLargeError(TabularRendererError): + + __TYPE = 'tabular_file_too_large' + + def __init__(self, message, *args, code: int=400, file_size: int=None, max_size: int=None, + **kwargs): + super().__init__(message, *args, code=code, **kwargs) + + self.file_size = file_size + self.max_size = max_size + + self.attr_stack.append([self.__TYPE, { + 'file_size': self.file_size, + 'max_size': self.max_size + }]) diff --git a/mfr/extensions/tabular/libs/xlrd_tools.py b/mfr/extensions/tabular/libs/xlrd_tools.py index fe4b19742..7afbd145d 100644 --- a/mfr/extensions/tabular/libs/xlrd_tools.py +++ b/mfr/extensions/tabular/libs/xlrd_tools.py @@ -19,7 +19,8 @@ def xlsx_xlrd(fp): for sheet in wb.sheets(): if sheet.ncols > max_size or sheet.nrows > max_size: - raise TableTooBigError('Table is too large to render.', '.xlsx') + raise TableTooBigError('Table is too large to render.', '.xlsx', + nbr_cols=sheet.ncols, nbr_rows=sheet.nrows) if sheet.ncols < 1 or sheet.nrows < 1: sheets[sheet.name] = ([], []) diff --git a/mfr/extensions/tabular/render.py b/mfr/extensions/tabular/render.py index d0a806909..b9d3cb1a6 100644 --- a/mfr/extensions/tabular/render.py +++ b/mfr/extensions/tabular/render.py @@ -1,12 +1,16 @@ -import os import json +import logging +import os +from humanfriendly import format_size from mako.lookup import TemplateLookup -from mfr.core import extension +from mfr.core import extension from mfr.extensions.tabular import settings from mfr.extensions.tabular import exceptions +logger = logging.getLogger(__name__) + class TabularRenderer(extension.BaseRenderer): @@ -16,6 +20,16 @@ class TabularRenderer(extension.BaseRenderer): ]).get_template('viewer.mako') def render(self): + file_size = os.path.getsize(self.file_path) + if file_size > settings.MAX_FILE_SIZE: + raise exceptions.FileTooLargeError( + 'Tabular files larger than {} are not rendered. Please download ' + 'the file to view.'.format(format_size(settings.MAX_FILE_SIZE, binary=True)), + file_size=file_size, + max_size=settings.MAX_FILE_SIZE, + extension=self.metadata.ext, + ) + with open(self.file_path, errors='replace') as fp: sheets, size = self._render_grid(fp, self.metadata.ext) return self.TEMPLATE.render( @@ -34,7 +48,7 @@ def file_required(self): def cache_result(self): return True - def _render_grid(self, fp, ext, *args, **kwargs): # assets_path, ext): + def _render_grid(self, fp, ext, *args, **kwargs): """Render a tabular file to html :param fp: file pointer object :return: RenderResult object containing html and assets @@ -46,14 +60,23 @@ def _render_grid(self, fp, ext, *args, **kwargs): # assets_path, ext): size = settings.SMALL_TABLE self._renderer_tabular_metrics['size'] = 'small' self._renderer_tabular_metrics['nbr_sheets'] = len(sheets) - for sheet in sheets: - sheet = sheets[sheet] # Sheets are stored in key-value pairs of the form {sheet: (col, row)} - if len(sheet[0]) > 9: # Check the number of columns + for sheet_title in sheets: + sheet = sheets[sheet_title] + + # sheet is a two-element list. sheet[0] is a list of dicts containing metadata about + # the column headers. Each dict contains four keys: `field`, `name`, `sortable`, `id`. + # sheet[1] is a list of dicts where each dict contains the row data. The keys are the + # fields the data belongs to and the values are the data values. + + nbr_cols = len(sheet[0]) + if nbr_cols > 9: size = settings.BIG_TABLE self._renderer_tabular_metrics['size'] = 'big' - if len(sheet[0]) > settings.MAX_SIZE or len(sheet[1]) > settings.MAX_SIZE: - raise exceptions.TableTooBigError('Table is too large to render.', extension=ext) + nbr_rows = len(sheet[1]) + if nbr_cols > settings.MAX_SIZE or nbr_rows > settings.MAX_SIZE: + raise exceptions.TableTooBigError('Table is too large to render.', extension=ext, + nbr_cols=nbr_cols, nbr_rows=nbr_rows) return sheets, size @@ -61,7 +84,7 @@ def _populate_data(self, fp, ext): """Determine the appropriate library and use it to populate rows and columns :param fp: file pointer :param ext: file extension - :return: tuple of column headers and row data + :return: a dict mapping sheet titles to tuples of column headers and row data """ function_preference = settings.LIBS.get(ext.lower()) diff --git a/mfr/extensions/tabular/settings.py b/mfr/extensions/tabular/settings.py index 258bafc03..c085bab26 100644 --- a/mfr/extensions/tabular/settings.py +++ b/mfr/extensions/tabular/settings.py @@ -4,7 +4,8 @@ config = settings.child('TABULAR_EXTENSION_CONFIG') -MAX_SIZE = int(config.get('MAX_SIZE', 10000)) +MAX_FILE_SIZE = int(config.get('MAX_FILE_SIZE', 10 * 1024 * 1024)) # 10Mb +MAX_SIZE = int(config.get('MAX_SIZE', 10000)) # max number of rows or columns allowed. TABLE_WIDTH = int(config.get('TABLE_WIDTH', 700)) TABLE_HEIGHT = int(config.get('TABLE_HEIGHT', 600))