From 0e14829e25d5593a9ceac29d8456c9d20cdacd5d Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Sep 2020 22:11:59 +0100 Subject: [PATCH 01/10] :sparkles: new query reader --- pyexcel_io/constants.py | 1 + pyexcel_io/database/__init__.py | 6 +++++- pyexcel_io/database/exporters/queryset.py | 16 ++++++++++++++++ pyexcel_io/database/querysets.py | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 pyexcel_io/database/exporters/queryset.py diff --git a/pyexcel_io/constants.py b/pyexcel_io/constants.py index f969f40..533a196 100644 --- a/pyexcel_io/constants.py +++ b/pyexcel_io/constants.py @@ -49,6 +49,7 @@ FILE_FORMAT_XLSM = "xlsm" DB_SQL = "sql" DB_DJANGO = "django" +DB_QUERYSET = "queryset" KEYWORD_TSV_DIALECT = "excel-tab" KEYWORD_LINE_TERMINATOR = "lineterminator" diff --git a/pyexcel_io/database/__init__.py b/pyexcel_io/database/__init__.py index e5763f7..f28f0d5 100644 --- a/pyexcel_io/database/__init__.py +++ b/pyexcel_io/database/__init__.py @@ -8,9 +8,13 @@ :license: New BSD License, see LICENSE for more details """ from pyexcel_io.plugins import NewIOPluginInfoChain -from pyexcel_io.constants import DB_SQL, DB_DJANGO +from pyexcel_io.constants import DB_SQL, DB_DJANGO, DB_QUERYSET NewIOPluginInfoChain(__name__).add_a_reader( + relative_plugin_class_path="exporters.queryset.QueryReader", + locations=["file", "memory", "content"], + file_types=[DB_QUERYSET], +).add_a_reader( relative_plugin_class_path="exporters.django.DjangoBookReader", locations=["file", "memory", "content"], file_types=[DB_DJANGO], diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py new file mode 100644 index 0000000..5afedd7 --- /dev/null +++ b/pyexcel_io/database/exporters/queryset.py @@ -0,0 +1,16 @@ +from pyexcel_io.plugin_api.abstract_reader import IReader +from pyexcel_io.database.querysets import QuerysetsReader + + +class QueryReader(IReader): + def __init__(self, query_sets, _, column_names=None, **keywords): + self.query_sets = query_sets + self.column_names = column_names + self.keywords = keywords + self.content_array = [ + QuerysetsReader( + self.query_sets, self.column_names, **self.keywords) + ] + + def read_sheet(self, index): + return self.content_array[index] diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 012427b..0080f30 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -17,6 +17,7 @@ class QuerysetsReader(Sheet): """ turn querysets into an array """ def __init__(self, query_sets, column_names): + self.name = 'pyexcel-queryset' self.__column_names = column_names self.__query_sets = query_sets From c267e89f13bee0a6235d1f629485907228c60319 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Sep 2020 22:20:30 +0100 Subject: [PATCH 02/10] :green_heart: add close implementation --- pyexcel_io/database/exporters/queryset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py index 5afedd7..74b8cbb 100644 --- a/pyexcel_io/database/exporters/queryset.py +++ b/pyexcel_io/database/exporters/queryset.py @@ -14,3 +14,7 @@ def __init__(self, query_sets, _, column_names=None, **keywords): def read_sheet(self, index): return self.content_array[index] + + + def close(): + pass From 9fe1048dc50387903babb74e898bce252c47a71d Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Sep 2020 22:25:25 +0100 Subject: [PATCH 03/10] :lipstick: update coding style --- pyexcel_io/database/exporters/queryset.py | 6 +++--- pyexcel_io/database/querysets.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py index 74b8cbb..3d39b9c 100644 --- a/pyexcel_io/database/exporters/queryset.py +++ b/pyexcel_io/database/exporters/queryset.py @@ -1,5 +1,5 @@ -from pyexcel_io.plugin_api.abstract_reader import IReader from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.plugin_api.abstract_reader import IReader class QueryReader(IReader): @@ -9,12 +9,12 @@ def __init__(self, query_sets, _, column_names=None, **keywords): self.keywords = keywords self.content_array = [ QuerysetsReader( - self.query_sets, self.column_names, **self.keywords) + self.query_sets, self.column_names, **self.keywords + ) ] def read_sheet(self, index): return self.content_array[index] - def close(): pass diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 0080f30..93b41de 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -17,7 +17,7 @@ class QuerysetsReader(Sheet): """ turn querysets into an array """ def __init__(self, query_sets, column_names): - self.name = 'pyexcel-queryset' + self.name = "pyexcel-queryset" self.__column_names = column_names self.__query_sets = query_sets From 52352819745fe03f716439902ec46749f1556826 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Sep 2020 22:29:40 +0100 Subject: [PATCH 04/10] :bug: update close as class method --- pyexcel_io/database/exporters/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexcel_io/database/exporters/queryset.py b/pyexcel_io/database/exporters/queryset.py index 3d39b9c..fd8ccdb 100644 --- a/pyexcel_io/database/exporters/queryset.py +++ b/pyexcel_io/database/exporters/queryset.py @@ -16,5 +16,5 @@ def __init__(self, query_sets, _, column_names=None, **keywords): def read_sheet(self, index): return self.content_array[index] - def close(): + def close(self): pass From 969d2c59c6f5554869652486703143ecd90869c6 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Sep 2020 22:56:35 +0100 Subject: [PATCH 05/10] :lipstick: use default sheet name --- pyexcel_io/database/querysets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 93b41de..4bc42fc 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -11,13 +11,14 @@ from itertools import chain from pyexcel_io.plugin_api.abstract_sheet import Sheet +from pyexcel_io.constants import DEFAULT_SHEET_NAME class QuerysetsReader(Sheet): """ turn querysets into an array """ def __init__(self, query_sets, column_names): - self.name = "pyexcel-queryset" + self.name = DEFAULT_SHEET_NAME self.__column_names = column_names self.__query_sets = query_sets From 53f5c84541a79558a24ddf9bfdca0c10a4449568 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 29 Sep 2020 10:04:54 +0100 Subject: [PATCH 06/10] :microscope: test query reader --- tests/test_sql_book.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index d37922b..34233bf 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -24,6 +24,7 @@ ) from sqlalchemy.ext.declarative import declarative_base from pyexcel_io.database.querysets import QuerysetsReader +from pyexcel_io.database.exporters.queryset import QueryReader from pyexcel_io.database.exporters.sqlalchemy import ( SQLBookReader, SQLTableReader, @@ -226,6 +227,13 @@ def test_one_table(self): reader = QuerysetsReader(query_sets, self.data[0]) results = reader.to_array() assert list(results) == self.results + + query_sets = mysession.query(Pyexcel).all() + reader = QueryReader(query_sets, None, column_names=self.data[0]) + result = reader.read_all() + for key in result: + result[key] = list(result[key]) + eq_(result, {'pyexcel_sheet1': self.results}) mysession.close() def test_update_existing_row(self): From ffe6005aab7791cd4c4af22f9e5f5014b35be179 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 29 Sep 2020 10:06:02 +0100 Subject: [PATCH 07/10] :lipstick: update coding style --- pyexcel_io/database/querysets.py | 2 +- tests/test_sql_book.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyexcel_io/database/querysets.py b/pyexcel_io/database/querysets.py index 4bc42fc..b1555f1 100644 --- a/pyexcel_io/database/querysets.py +++ b/pyexcel_io/database/querysets.py @@ -10,8 +10,8 @@ import datetime from itertools import chain -from pyexcel_io.plugin_api.abstract_sheet import Sheet from pyexcel_io.constants import DEFAULT_SHEET_NAME +from pyexcel_io.plugin_api.abstract_sheet import Sheet class QuerysetsReader(Sheet): diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 34233bf..1af681f 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -233,7 +233,7 @@ def test_one_table(self): result = reader.read_all() for key in result: result[key] = list(result[key]) - eq_(result, {'pyexcel_sheet1': self.results}) + eq_(result, {"pyexcel_sheet1": self.results}) mysession.close() def test_update_existing_row(self): From c6f774f928986b5bf1b4acd812798da4374ea686 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 29 Sep 2020 18:56:33 +0100 Subject: [PATCH 08/10] :green_heart: test close method --- tests/test_sql_book.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py index 1af681f..334a781 100644 --- a/tests/test_sql_book.py +++ b/tests/test_sql_book.py @@ -229,11 +229,12 @@ def test_one_table(self): assert list(results) == self.results query_sets = mysession.query(Pyexcel).all() - reader = QueryReader(query_sets, None, column_names=self.data[0]) - result = reader.read_all() + query_reader = QueryReader(query_sets, None, column_names=self.data[0]) + result = query_reader.read_all() for key in result: result[key] = list(result[key]) eq_(result, {"pyexcel_sheet1": self.results}) + query_reader.close() mysession.close() def test_update_existing_row(self): From 9261a11128eaa3358d01c68f21c92b4c451e88ab Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 29 Sep 2020 19:02:48 +0100 Subject: [PATCH 09/10] :books: update change log --- changelog.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.yml b/changelog.yml index 58ce90b..42f52b2 100644 --- a/changelog.yml +++ b/changelog.yml @@ -12,6 +12,8 @@ releases: details: - pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted. please read 'Packaging with Pyinstaller' in the documentation. + - new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting query set + source done. it is against the plugin interface. version: 0.6.0 date: tbd - changes: From 08e4bd8ab6855245e7ed40e1032667821d1dc69f Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 29 Sep 2020 18:03:26 +0000 Subject: [PATCH 10/10] This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37abb53..0a57e54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ Change log #. pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted. please read 'Packaging with Pyinstaller' in the documentation. +#. new query set reader plugin. pyexcel<=0.6.4 has used intrusive way of getting + query set source done. it is against the plugin interface. 0.5.20 - 17.7.2019 --------------------------------------------------------------------------------