diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py index 0717cfd..77defa4 100644 --- a/pyexcel_io/database/exporters/django.py +++ b/pyexcel_io/database/exporters/django.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader class DjangoModelReader(QuerysetsReader): @@ -26,7 +27,7 @@ def __init__(self, model, export_columns=None, **keywords): ) -class DjangoBookReader(object): +class DjangoBookReader(IReader): """ read django models """ def __init__(self, exporter, _, **keywords): diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py index 077cb21..0c31e2e 100644 --- a/pyexcel_io/database/exporters/sqlalchemy.py +++ b/pyexcel_io/database/exporters/sqlalchemy.py @@ -8,6 +8,7 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader class SQLTableReader(QuerysetsReader): @@ -30,7 +31,7 @@ def __init__(self, session, table, export_columns=None, **keywords): QuerysetsReader.__init__(self, everything, column_names, **keywords) -class SQLBookReader(object): +class SQLBookReader(IReader): """ read a table via sqlalchemy """ def __init__(self, exporter, _, **keywords): diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 61640d4..012427b 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -10,17 +10,19 @@ import datetime from itertools import chain -from pyexcel_io.sheet import SheetReader +from pyexcel_io.plugin_api.abstract_sheet import Sheet -class QuerysetsReader(SheetReader): +class QuerysetsReader(Sheet): """ turn querysets into an array """ - def __init__(self, query_sets, column_names, **keywords): - SheetReader.__init__(self, query_sets, **keywords) + def __init__(self, query_sets, column_names): self.__column_names = column_names self.__query_sets = query_sets + def row_iterator(self): + return chain([self.__column_names], self.__query_sets) + def to_array(self): """ Convert query sets into an array @@ -28,12 +30,9 @@ def to_array(self): if len(self.__query_sets) == 0: yield [] - for element in SheetReader.to_array(self): + for element in Sheet.to_array(self): yield element - def row_iterator(self): - return chain([self.__column_names], self.__query_sets) - def column_iterator(self, row): if self.__column_names is None: return diff --git a/pyexcel_io/io.py b/pyexcel_io/io.py index 0674a63..84b569d 100644 --- a/pyexcel_io/io.py +++ b/pyexcel_io/io.py @@ -184,6 +184,8 @@ def load_data( reader.open_content(file_content, **keywords) elif file_stream: reader.open_stream(file_stream, **keywords) + else: + raise IOError("Unrecognized options") if sheet_name: result = reader.read_sheet_by_name(sheet_name) elif sheet_index is not None: diff --git a/pyexcel_io/plugin_api/__init__.py b/pyexcel_io/plugin_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyexcel_io/plugin_api/abstract_reader.py b/pyexcel_io/plugin_api/abstract_reader.py new file mode 100644 index 0000000..bd3d1ee --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_reader.py @@ -0,0 +1,9 @@ +from pyexcel_io._compact import OrderedDict + + +class IReader(object): + def read_all(self): + result = OrderedDict() + for index, sheet in enumerate(self.content_array): + result.update({sheet.name: self.read_sheet(index).to_array()}) + return result diff --git a/pyexcel_io/plugin_api/abstract_sheet.py b/pyexcel_io/plugin_api/abstract_sheet.py new file mode 100644 index 0000000..e4fcc01 --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_sheet.py @@ -0,0 +1,9 @@ +class Sheet(object): + def to_array(self): + data = [] + for row in self.row_iterator(): + my_row = [] + for element in self.column_iterator(row): + my_row.append(element) + data.append(my_row) + return data diff --git a/pyexcel_io/plugin_api/abstract_writer.py b/pyexcel_io/plugin_api/abstract_writer.py new file mode 100644 index 0000000..08d7e40 --- /dev/null +++ b/pyexcel_io/plugin_api/abstract_writer.py @@ -0,0 +1,9 @@ +class IWriter(object): + def write(self, incoming_dict): + for sheet_name in incoming_dict: + sheet_writer = self.create_sheet(sheet_name) + if sheet_writer: + sheet_writer.write_array(incoming_dict[sheet_name]) + sheet_writer.close() + else: + raise Exception("Cannot create a sheet writer!") diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py index 535f8be..9e78b12 100644 --- a/pyexcel_io/readers/csv_in_file.py +++ b/pyexcel_io/readers/csv_in_file.py @@ -5,11 +5,12 @@ from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io.readers.csv_sheet import CSVFileReader +from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_NEWLINE = "\r\n" -class FileReader(object): +class FileReader(IReader): def __init__(self, file_name, file_type, **keywords): """Load content from a file :params str filename: an accessible file path diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py index 3e39521..02fdc04 100644 --- a/pyexcel_io/readers/csv_in_memory.py +++ b/pyexcel_io/readers/csv_in_memory.py @@ -4,11 +4,12 @@ from pyexcel_io import constants from pyexcel_io.sheet import NamedContent from pyexcel_io.readers.csv_sheet import CSVinMemoryReader +from pyexcel_io.plugin_api.abstract_reader import IReader DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s" -class MemoryReader(object): +class MemoryReader(IReader): def __init__( self, file_stream, file_type, multiple_sheets=False, **keywords ): diff --git a/pyexcel_io/readers/csv_sheet.py b/pyexcel_io/readers/csv_sheet.py index 9eb72a4..5c58ef4 100644 --- a/pyexcel_io/readers/csv_sheet.py +++ b/pyexcel_io/readers/csv_sheet.py @@ -12,6 +12,7 @@ import pyexcel_io.service as service import pyexcel_io._compact as compact import pyexcel_io.constants as constants +from pyexcel_io.sheet import SheetReader DEFAULT_SEPARATOR = "__" DEFAULT_SHEET_SEPARATOR_FORMATTER = "---%s---" % constants.DEFAULT_NAME + "%s" @@ -91,7 +92,7 @@ def close(self): pass -class CSVSheetReader(object): +class CSVSheetReader(SheetReader): """ generic csv file reader""" def __init__( @@ -107,6 +108,7 @@ def __init__( default_float_nan=None, **keywords ): + SheetReader.__init__(self, sheet, **keywords) self._native_sheet = sheet self._keywords = keywords self._encoding = encoding diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py index f435afd..9fdc55e 100644 --- a/pyexcel_io/readers/csvz.py +++ b/pyexcel_io/readers/csvz.py @@ -14,9 +14,10 @@ from pyexcel_io.sheet import NamedContent from pyexcel_io._compact import StringIO from pyexcel_io.readers.csv_sheet import CSVinMemoryReader +from pyexcel_io.plugin_api.abstract_reader import IReader -class FileReader(object): +class FileReader(IReader): def __init__(self, file_alike_object, file_type, **keywords): self.content_array = [] try: diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py index 111b74c..7a88002 100644 --- a/pyexcel_io/writers/csv_in_file.py +++ b/pyexcel_io/writers/csv_in_file.py @@ -1,8 +1,9 @@ from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVFileWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvFileWriter: +class CsvFileWriter(IWriter): def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords @@ -21,15 +22,6 @@ def create_sheet(self, name): self.__index = self.__index + 1 return self.writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): if self.writer: self.writer.close() diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py index 9e04b9d..ff0e6cb 100644 --- a/pyexcel_io/writers/csv_in_memory.py +++ b/pyexcel_io/writers/csv_in_memory.py @@ -1,8 +1,9 @@ from pyexcel_io import constants from pyexcel_io.writers.csv_sheet import CSVMemoryWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvMemoryWriter: +class CsvMemoryWriter(IWriter): def __init__(self, file_alike_object, file_type, **keywords): self._file_alike_object = file_alike_object self._keywords = keywords @@ -21,14 +22,5 @@ def create_sheet(self, name): self.__index = self.__index + 1 return writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): pass diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py index ace68c9..0e0379b 100644 --- a/pyexcel_io/writers/csvz_writer.py +++ b/pyexcel_io/writers/csvz_writer.py @@ -2,9 +2,10 @@ from pyexcel_io import constants from pyexcel_io.writers.csvz_sheet import CSVZipSheetWriter +from pyexcel_io.plugin_api.abstract_writer import IWriter -class CsvZipWriter(object): +class CsvZipWriter(IWriter): """ csvz writer @@ -29,15 +30,6 @@ def create_sheet(self, name): ) return writer - def write(self, incoming_dict): - for sheet_name in incoming_dict: - sheet_writer = self.create_sheet(sheet_name) - if sheet_writer: - sheet_writer.write_array(incoming_dict[sheet_name]) - sheet_writer.close() - else: - raise Exception("Cannot create a sheet writer!") - def close(self): if self.zipfile: self.zipfile.close() diff --git a/tests/test_csv_book.py b/tests/test_csv_book.py index 381ca69..c23489f 100644 --- a/tests/test_csv_book.py +++ b/tests/test_csv_book.py @@ -4,7 +4,6 @@ import pyexcel_io.manager as manager from pyexcel_io.sheet import NamedContent -from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import BytesIO, StringIO from pyexcel_io.readers.csv_sheet import ( CSVFileReader, @@ -32,9 +31,7 @@ def test_sheet_reader(self): sheet.get_file_handle() def test_sheet_file_reader(self): - r = EncapsulatedSheetReader( - CSVFileReader(NamedContent(self.file_type, self.test_file)) - ) + r = CSVFileReader(NamedContent(self.file_type, self.test_file)) result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -43,9 +40,8 @@ def test_sheet_memory_reader(self): with open(self.test_file, "r") as f: io.write(f.read()) io.seek(0) - r = EncapsulatedSheetReader( - CSVinMemoryReader(NamedContent(self.file_type, io)) - ) + r = CSVinMemoryReader(NamedContent(self.file_type, io)) + result = list(r.to_array()) self.assertEqual(result, self.expected_data) @@ -112,9 +108,7 @@ def setUp(self): f.write(",".join(row) + "\n") def test_sheet_file_reader(self): - r = EncapsulatedSheetReader( - CSVFileReader(NamedContent(self.file_type, self.test_file)) - ) + r = CSVFileReader(NamedContent(self.file_type, self.test_file)) result = list(r.to_array()) self.assertEqual(result, [[1], [4, 5, 6], ["", 7]]) @@ -124,9 +118,8 @@ def tearDown(self): def test_utf16_decoding(): test_file = os.path.join("tests", "fixtures", "csv-encoding-utf16.csv") - reader = EncapsulatedSheetReader( - CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") - ) + reader = CSVFileReader(NamedContent("csv", test_file), encoding="utf-16") + content = list(reader.to_array()) expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) @@ -149,9 +142,10 @@ def test_utf16_encoding(): def test_utf16_memory_decoding(): test_content = u"Äkkilähdöt,Matkakirjoituksia,Matkatoimistot" test_content = BytesIO(test_content.encode("utf-16")) - reader = EncapsulatedSheetReader( - CSVinMemoryReader(NamedContent("csv", test_content), encoding="utf-16") + reader = CSVinMemoryReader( + NamedContent("csv", test_content), encoding="utf-16" ) + content = list(reader.to_array()) expected = [["Äkkilähdöt", "Matkakirjoituksia", "Matkatoimistot"]] eq_(content, expected) diff --git a/tests/test_django_book.py b/tests/test_django_book.py index b92356c..fa231c4 100644 --- a/tests/test_django_book.py +++ b/tests/test_django_book.py @@ -1,4 +1,5 @@ from pyexcel_io import save_data +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import OrderedDict from pyexcel_io.constants import DB_DJANGO from pyexcel_io.database.common import ( @@ -210,7 +211,9 @@ def row_renderer(row): return [str(element) for element in row] # the key point of this test case - reader = DjangoModelReader(model, row_renderer=row_renderer) + reader = EncapsulatedSheetReader( + DjangoModelReader(model), row_renderer=row_renderer + ) data = reader.to_array() expected = [["X", "Y", "Z"], ["1", "2", "3"], ["4", "5", "6"]] eq_(list(data), expected) @@ -336,16 +339,10 @@ def test_reading_from_more_models(self): exporter.append(adapter1) exporter.append(adapter2) reader = DjangoBookReader(exporter, "django") - result = OrderedDict() - for index, sheet in enumerate(reader.content_array): - result.update( - { - reader.content_array[index].name: list( - reader.read_sheet(index).to_array() - ) - } - ) - assert result == self.content + result = reader.read_all() + for key in result: + result[key] = list(result[key]) + eq_(result, self.content) @raises(Exception) def test_special_case_where_only_one_model_used(self): @@ -377,25 +374,33 @@ def setUp(self): self.model._meta.update(["X", "Y", "Z"]) def test_load_sheet_from_django_model_with_filter(self): - reader = DjangoModelReader(self.model, start_row=0, row_limit=2) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_row=0, row_limit=2 + ) data = reader.to_array() expected = [["X", "Y", "Z"], [1, 2, 3]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_1(self): - reader = DjangoModelReader(self.model, start_row=1, row_limit=3) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_row=1, row_limit=3 + ) data = reader.to_array() expected = [[1, 2, 3], [4, 5, 6]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_2(self): - reader = DjangoModelReader(self.model, start_column=1) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_column=1 + ) data = reader.to_array() expected = [["Y", "Z"], [2, 3], [5, 6]] eq_(list(data), expected) def test_load_sheet_from_django_model_with_filter_3(self): - reader = DjangoModelReader(self.model, start_column=1, column_limit=1) + reader = EncapsulatedSheetReader( + DjangoModelReader(self.model), start_column=1, column_limit=1 + ) data = reader.to_array() expected = [["Y"], [2], [5]] eq_(list(data), expected) diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index e0c8b07..d37922b 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -14,6 +14,7 @@ create_engine, ) from sqlalchemy.orm import backref, relationship, sessionmaker +from pyexcel_io.reader import EncapsulatedSheetReader from pyexcel_io._compact import OrderedDict from pyexcel_io.database.common import ( SQLTableExporter, @@ -124,7 +125,7 @@ def test_sql(self): ["2014-11-12", 1, "Smith", 12.25], ] # 'pyexcel' here is the table name - assert list(data) == content + eq_(list(data), content) mysession.close() def test_sql_formating(self): @@ -134,8 +135,8 @@ def custom_renderer(row): return [str(element) for element in row] # the key for this test case - sheet = SQLTableReader( - mysession, Pyexcel, row_renderer=custom_renderer + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), row_renderer=custom_renderer ) data = sheet.to_array() content = [ @@ -148,7 +149,9 @@ def custom_renderer(row): def test_sql_filter(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_row=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_row=1 + ) data = sheet.to_array() content = [ ["2014-11-11", 0, "Adam", 11.25], @@ -160,7 +163,9 @@ def test_sql_filter(self): def test_sql_filter_1(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_row=1, row_limit=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_row=1, row_limit=1 + ) data = sheet.to_array() content = [["2014-11-11", 0, "Adam", 11.25]] # 'pyexcel'' here is the table name @@ -169,7 +174,9 @@ def test_sql_filter_1(self): def test_sql_filter_2(self): mysession = Session() - sheet = SQLTableReader(mysession, Pyexcel, start_column=1) + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_column=1 + ) data = sheet.to_array() content = [ ["id", "name", "weight"], @@ -182,8 +189,8 @@ def test_sql_filter_2(self): def test_sql_filter_3(self): mysession = Session() - sheet = SQLTableReader( - mysession, Pyexcel, start_column=1, column_limit=1 + sheet = EncapsulatedSheetReader( + SQLTableReader(mysession, Pyexcel), start_column=1, column_limit=1 ) data = sheet.to_array() content = [["id"], [0], [1]] @@ -455,15 +462,10 @@ def test_read(self): post_adapter = SQLTableExportAdapter(Post) exporter.append(post_adapter) reader = SQLBookReader(exporter, "sql") - result = OrderedDict() - for index, sheet in enumerate(reader.content_array): - result.update( - { - reader.content_array[index].name: list( - reader.read_sheet(index).to_array() - ) - } - ) + result = reader.read_all() + for key in result: + result[key] = list(result[key]) + assert json.dumps(result) == ( '{"category": [["id", "name"], [1, "News"], [2, "Sports"]], ' + '"post": [["body", "category_id", "id", "pub_date", "title"], '