From 8e80ecd3a0f6154af62637df7532f69e2a5b54d5 Mon Sep 17 00:00:00 2001 From: begoat Date: Mon, 2 Aug 2021 13:11:59 +0800 Subject: [PATCH 1/5] feat(data.py): add custom datareader --- pandas_datareader/__init__.py | 4 +++ pandas_datareader/data.py | 68 +++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/pandas_datareader/__init__.py b/pandas_datareader/__init__.py index a792a806..616a75b9 100644 --- a/pandas_datareader/__init__.py +++ b/pandas_datareader/__init__.py @@ -28,6 +28,8 @@ get_records_iex, get_summary_iex, get_tops_iex, + get_custom_datareader, + register_custom_datareader, ) PKG = os.path.dirname(__file__) @@ -62,6 +64,8 @@ "get_data_tiingo", "get_iex_data_tiingo", "get_data_alphavantage", + "get_custom_datareader", + "register_custom_datareader", "test", ] diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py index c2d6223a..4e4a1d8a 100644 --- a/pandas_datareader/data.py +++ b/pandas_datareader/data.py @@ -7,7 +7,7 @@ import warnings from pandas.util._decorators import deprecate_kwarg - +from pandas_datareader.base import _BaseReader from pandas_datareader.av.forex import AVForexReader from pandas_datareader.av.quotes import AVQuotesReader from pandas_datareader.av.sector import AVSectorPerformanceReader @@ -60,9 +60,11 @@ "get_iex_book", "get_dailysummary_iex", "get_data_stooq", + "register_custom_datareader", "DataReader", ] +custom_datareader = {} def get_data_alphavantage(*args, **kwargs): return AVTimeSeriesReader(*args, **kwargs).read() @@ -270,6 +272,55 @@ def get_iex_book(*args, **kwargs): return IEXDeep(*args, **kwargs).read() +def register_custom_datareader(custom_name, custom_class): + """ + Registers a custom datareader to be used + + Parameters + ---------- + custom_name : str + A string represents name you want to give to your custom class + custom_class : _BaseReader + A class that extends _BaseReader + + Returns + ------- + True if successful, Error otherwise. + """ + custom_datareader[custom_name] = custom_class + return True + +def unregister_custom_datareader(custom_name): + """ + Unregisters a custom datareader to be used + + Parameters + ---------- + custom_name : str + A string represents name you want to give to your custom class + + Returns + ------- + True if successful, Error otherwise. + """ + del custom_datareader[custom_name] + return True + +def get_custom_datareader(custom_name): + """ + Get a custom datareader registered before + + Parameters + ---------- + custom_name : str + A string represents name you gave to your custom class + + Returns + ------- + Class registered before + """ + return custom_datareader[custom_name] + @deprecate_kwarg("access_key", "api_key") def DataReader( name, @@ -329,6 +380,7 @@ def DataReader( ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") """ + custom_source = list(custom_datareader.keys()) expected_source = [ "yahoo", "iex", @@ -360,7 +412,7 @@ def DataReader( "av-intraday", "econdb", "naver", - ] + ] + custom_source if data_source not in expected_source: msg = "data_source=%r is not implemented" % data_source @@ -668,6 +720,18 @@ def DataReader( session=session, ).read() + elif data_source in custom_source: + CustomDataReader = get_custom_datareader(data_source) + return CustomDataReader( + symbols=name, + start=start, + end=end, + retry_count=retry_count, + pause=pause, + session=session, + api_key=api_key, + ).read() + else: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg) From 8aeac36384de717a5d79f8fd5ff3286e60135639 Mon Sep 17 00:00:00 2001 From: begoat Date: Mon, 2 Aug 2021 14:54:35 +0800 Subject: [PATCH 2/5] refactor(data.py): code style check --- pandas_datareader/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py index 4e4a1d8a..4dc78348 100644 --- a/pandas_datareader/data.py +++ b/pandas_datareader/data.py @@ -66,6 +66,7 @@ custom_datareader = {} + def get_data_alphavantage(*args, **kwargs): return AVTimeSeriesReader(*args, **kwargs).read() @@ -290,6 +291,7 @@ def register_custom_datareader(custom_name, custom_class): custom_datareader[custom_name] = custom_class return True + def unregister_custom_datareader(custom_name): """ Unregisters a custom datareader to be used @@ -306,6 +308,7 @@ def unregister_custom_datareader(custom_name): del custom_datareader[custom_name] return True + def get_custom_datareader(custom_name): """ Get a custom datareader registered before @@ -321,6 +324,7 @@ def get_custom_datareader(custom_name): """ return custom_datareader[custom_name] + @deprecate_kwarg("access_key", "api_key") def DataReader( name, From bb67ab10b724f2d4fb5f17616d020c13ef875726 Mon Sep 17 00:00:00 2001 From: begoat Date: Mon, 2 Aug 2021 14:55:14 +0800 Subject: [PATCH 3/5] test(test_data.py): add testcase for custom datasource --- pandas_datareader/tests/test_data.py | 43 +++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py index 4cdea1d0..6349c42e 100644 --- a/pandas_datareader/tests/test_data.py +++ b/pandas_datareader/tests/test_data.py @@ -1,7 +1,8 @@ from pandas import DataFrame import pytest -from pandas_datareader.data import DataReader +from pandas_datareader.base import _DailyBaseReader +from pandas_datareader.data import DataReader, register_custom_datareader pytestmark = pytest.mark.stable @@ -18,3 +19,43 @@ def test_read_fred(self): def test_not_implemented(self): with pytest.raises(NotImplementedError): DataReader("NA", "NA") + + def test_custom_reader_acc(self): + class DemoReader(_DailyBaseReader): + def __init__( + self, + symbols=None, + start=None, + end=None, + retry_count=3, + pause=0.1, + session=None, + api_key=None, + ): + super().__init__( + symbols=symbols, + start=start, + end=end, + retry_count=retry_count, + pause=pause, + session=session, + ) + + @property + def url(self): + return 'https://stooq.com/q/d/l/' + + def _get_params(self, symbol): + params = { + 's': symbol, + 'i': "d" + } + return params + + register_custom_datareader('demo', DemoReader) + result = DataReader('USDJPY', 'demo') + assert isinstance(result, DataFrame) + + def test_custom_reader_fail(self): + with pytest.raises(NotImplementedError): + DataReader('USDJPY', 'demo1') \ No newline at end of file From d0f6a5a42ab8223cbbf29b9152eceaf7f1e10a03 Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 2 Aug 2021 20:34:16 +0800 Subject: [PATCH 4/5] refactor(test_data.py): add blank line at eof --- pandas_datareader/tests/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py index 6349c42e..db1419cc 100644 --- a/pandas_datareader/tests/test_data.py +++ b/pandas_datareader/tests/test_data.py @@ -58,4 +58,4 @@ def _get_params(self, symbol): def test_custom_reader_fail(self): with pytest.raises(NotImplementedError): - DataReader('USDJPY', 'demo1') \ No newline at end of file + DataReader('USDJPY', 'demo1') From a1e7a8e435a4f71d93d3cf1bdc052e97c61afa28 Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 2 Aug 2021 20:35:48 +0800 Subject: [PATCH 5/5] refactor(test_data.py): black check test_file --- pandas_datareader/tests/test_data.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py index db1419cc..0a77a5e3 100644 --- a/pandas_datareader/tests/test_data.py +++ b/pandas_datareader/tests/test_data.py @@ -43,19 +43,16 @@ def __init__( @property def url(self): - return 'https://stooq.com/q/d/l/' + return "https://stooq.com/q/d/l/" def _get_params(self, symbol): - params = { - 's': symbol, - 'i': "d" - } + params = {"s": symbol, "i": "d"} return params - register_custom_datareader('demo', DemoReader) - result = DataReader('USDJPY', 'demo') + register_custom_datareader("demo", DemoReader) + result = DataReader("USDJPY", "demo") assert isinstance(result, DataFrame) def test_custom_reader_fail(self): with pytest.raises(NotImplementedError): - DataReader('USDJPY', 'demo1') + DataReader("USDJPY", "demo1")