diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d18a98d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.md LICENSE requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index d704f34..58da2a9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,25 @@ +# pystockdb + [![Build Status](https://travis-ci.org/portfolioplus/pystockdb.svg?branch=master)](https://travis-ci.org/portfolioplus/pystockdb) [![Coverage Status](https://coveralls.io/repos/github/portfolioplus/pystockdb/badge.svg?branch=master)](https://coveralls.io/github/portfolioplus/pystockdb?branch=master) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/07e6231a5a8c415a9f27736e02a286da)](https://www.codacy.com/app/SlashGordon/pystockdb?utm_source=github.com&utm_medium=referral&utm_content=portfolioplus/pystockdb&utm_campaign=Badge_Grade) -# pystockdb +Database for stocks based on [pony.orm](https://github.com/ponyorm/pony). +This package provides an create, sync and update tool. -database for stocks based on pony.orm: +At the moment we are only support a few stocks. +If you want to have more, please contribute [pytickersymbols](https://github.com/portfolioplus/pytickersymbols). -# quick start +## install + +```shell +pip install pystockdb +``` + +## quick start + +In all samples we use sqlite but you are free to use other providers. +For more information's please read [Connecting to the Database](https://docs.ponyorm.org/database.html). Install sqlite stock db: @@ -20,13 +34,55 @@ config = { 'currency': 'EUR', 'db_args': { 'provider': 'sqlite', - 'filename': db_name, + 'filename': 'demo.sqlite', 'create_db': True }, } create = CreateAndFillDataBase(config, logger) create.build() ``` -# issue tracker + +Update sqlite stock db: + +```python +import logging +from pystockdb.tools.update import UpdateDataBaseStocks + +logger = logging.getLogger('test') +config = { + 'symbols': ['ALL'], + 'prices': True, # update prices + 'fundamentals': True, # update fundamental stock data + 'db_args': { + 'provider': 'sqlite', + 'filename': 'demo.sqlite', + 'create_db': False + }, +} +update = UpdateDataBaseStocks(config, logger) +update.build() +``` + +Sync sqlite stock db: + +```python +import logging +from pystockdb.tools.sync import SyncDataBaseStocks + +logger = logging.getLogger('test') +config = { + 'max_history': 1, + 'indices': ['CAC 40'], # add new index to existing database + 'currency': 'EUR', + 'db_args': { + 'provider': 'sqlite', + 'filename': 'demo.sqlite', + }, +} +sync = SyncDataBaseStocks(config, logger) +sync.build() +``` + +## issue tracker [https://github.com/portfolioplus/pystockdb/issuese](https://github.com/portfolioplus/pystockdb/issues") diff --git a/requirements.txt b/requirements.txt index 7b7406f..212df31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -pytickersymbols==1.0.8 -yfinance==0.1.43 +pytickersymbols==1.1.3 +pandas==0.24.2 +yfinance==0.1.44 uplink==0.9.0 -pony==0.7.10 \ No newline at end of file +pony==0.7.10 diff --git a/setup.py b/setup.py index c570894..bfa1569 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ Copyright 2019 Slash Gordon - Use of this source code is governed by a GNU General Public License v3 or + Use of this source code is governed by a GNU General Public License v3 or later that can be found in the LICENSE file. """ @@ -14,34 +14,33 @@ EXCLUDE_FROM_PACKAGES = ['test', 'test.*', 'test*'] VERSION = '0.0.1' - -def get_requirements(requirements): - with open(requirements) as requirement_file: - content = requirement_file.readlines() - content = [x.strip() for x in content] - return content - - with open("README.md", "r") as fh: long_description = fh.read() +INSTALL_REQUIRES = ( + [ + 'pytickersymbols>=1.1.3', 'pandas==0.24.2', 'yfinance>=0.1.44', + 'uplink>=0.9.0', 'pony==0.7.10' + ] +) + setup( name="pystockdb", version=VERSION, author="Slash Gordon", - author_email="slash.gordon.dev@gmail.com ", - py_modules=['pystockdb'], + author_email="slash.gordon.dev@gmail.com", package_dir={'': 'src'}, description="Simple stock db with tools.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/portfolioplus/pystockdb", + install_requires=INSTALL_REQUIRES, packages=find_packages('src', exclude=EXCLUDE_FROM_PACKAGES), classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Natural Language :: English', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', @@ -53,4 +52,4 @@ def get_requirements(requirements): 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Office/Business :: Financial :: Investment', ], -) \ No newline at end of file +) diff --git a/src/pystockdb/db/schema/stocks.py b/src/pystockdb/db/schema/stocks.py index 1f65ac8..aca05ad 100644 --- a/src/pystockdb/db/schema/stocks.py +++ b/src/pystockdb/db/schema/stocks.py @@ -163,9 +163,9 @@ class Argument(db.Entity): class Data(db.Entity): id = PrimaryKey(int, auto=True) - date = Required(datetime, default=lambda: datetime.now()) + date = Required(datetime, default=datetime.now()) data = Required(Json) - md5 = Required(str) + hash = Required(str) data_item = Required('DataItem') diff --git a/src/pystockdb/tools/base.py b/src/pystockdb/tools/base.py index 11dcbcb..6598a7d 100644 --- a/src/pystockdb/tools/base.py +++ b/src/pystockdb/tools/base.py @@ -11,6 +11,8 @@ from pony.orm import commit, db_session +from pytickersymbols import PyTickerSymbols + from pystockdb.db.schema.stocks import (Index, Item, PriceItem, Stock, Tag, Type, db) from pystockdb.tools.data_crawler import DataCrawler @@ -25,11 +27,12 @@ def __init__(self, arguments: dict, logger: logging.Logger): self.logger = logger self.db_args = arguments["db_args"] self.arguments = arguments + self.ticker_symbols = PyTickerSymbols() # has connection if db.provider is None: # prepare db db.bind(**self.db_args) - if self.arguments.get('create', False): + if self.db_args.get('create_db', False): db.generate_mapping(check_tables=False) db.drop_all_tables(with_all_data=True) db.create_tables() @@ -75,7 +78,6 @@ def __add_stock_to_index(self, index, stock_info): stock = Stock(name=stock_info['name'], price_item=PriceItem(item=Item())) # add symbols - "".startswith yao = Tag.get(name=Tag.YAO) gog = Tag.get(name=Tag.GOG) usd = Tag.get(name=Tag.USD) @@ -117,6 +119,10 @@ def download_historicals(self, symbols, start, end): ids = [symbol.name for symbol in chunk] series = crawler.get_series_stack(ids, start=start, end=end) for symbol in chunk: + self.logger.debug( + 'Add prices for {} from {} until {}.'.format(symbol.name, + start, end) + ) for value in series[symbol.name]: symbol.prices.create(**value) commit() diff --git a/src/pystockdb/tools/create.py b/src/pystockdb/tools/create.py index e36a002..787416c 100644 --- a/src/pystockdb/tools/create.py +++ b/src/pystockdb/tools/create.py @@ -11,10 +11,7 @@ import logging from datetime import timedelta -from pony.orm import commit, db_session -from pytickersymbols import PyTickerSymbols - -from pystockdb.db.schema.stocks import Symbol, Tag, Type +from pystockdb.db.schema.stocks import Symbol, Tag from pystockdb.tools.base import DBBase @@ -29,7 +26,6 @@ def __init__(self, arguments: dict, logger: logging.Logger): self.currency = arguments['currency'] self.history = arguments['max_history'] self.indices_list = arguments['indices'] - self.ticker_symbols = PyTickerSymbols() def build(self): """ diff --git a/src/pystockdb/tools/fundamentals.py b/src/pystockdb/tools/fundamentals.py index 82ae302..502f34e 100644 --- a/src/pystockdb/tools/fundamentals.py +++ b/src/pystockdb/tools/fundamentals.py @@ -121,12 +121,12 @@ def get_income(self, ticker_id): @returns.json @get('/api/securities/stock/{ticker_id}/statementsV2Detail') - def get_sheet(self, ticker_id, type: Query, + def get_sheet(self, ticker_id, sheet_type: Query("type"), query_number: Query('queryNumber') = 300): """ Returns financial sheet(income, balance, cash flow) data for stock :param ticker_id: ticker id - :param type: income=1, balance=2 and cashflow = 3 + :param sheet_type: income=1, balance=2 and cashflow = 3 :param queryNumber: :return: """ diff --git a/src/pystockdb/tools/update.py b/src/pystockdb/tools/update.py index 74fc7c9..3086054 100644 --- a/src/pystockdb/tools/update.py +++ b/src/pystockdb/tools/update.py @@ -68,7 +68,7 @@ def update_prices(self): for key in update: start = datetime.datetime.strptime(key, '%Y-%m-%d') end = datetime.datetime.now() - if start >= end: + if start.date() >= end.date(): continue self.download_historicals(update[key], start=start.strftime('%Y-%m-%d'), @@ -89,7 +89,16 @@ def update_fundamentals(self): fundamentals = Fundamentals(base_url=Fundamentals.BASE_URL) tickers = fundamentals.get_ticker_ids(gog_syms) for ticker in tickers: - stock = [sto[0] for sto in stocks if sto[1] == ticker][0] + self.logger.debug( + 'Download fundamentals for {}'.format(tickers[ticker]) + ) + stock = [sto[0] for sto in stocks if sto[1] == ticker] + if len(stock) != 1: + self.logger.warning( + 'Can not download fundamentals for {}'.format(ticker) + ) + continue + stock = stock[0] ica = fundamentals.get_income_analysis(tickers[ticker]) ifc = fundamentals.get_income_facts(tickers[ticker]) rec = fundamentals.get_recommendation(tickers[ticker]) @@ -99,16 +108,16 @@ def update_fundamentals(self): for val in [(ica, Tag.ICA), (ifc, Tag.ICF), (rec, Tag.REC), (ble, Tag.BLE), (ico, Tag.ICO), (csh, Tag.CSH)]: # hash stock name, tag and data - m = hashlib.md5() + m = hashlib.sha256() m.update(val[1].encode('UTF-8')) m.update(stock.name.encode('UTF-8')) data_str = json.dumps(val[0]) m.update(data_str.encode('UTF-8')) - md5 = m.hexdigest() + shahash = m.hexdigest() # only add data if not exist - if Data.get(md5=md5) is None: + if Data.get(hash=shahash) is None: tag = Tag.get(name=val[1]) - obj = Data(data=val[0], md5=md5, + obj = Data(data=val[0], hash=shahash, data_item=DataItem(item=Item(tags=[tag]))) stock.data_items.add(obj.data_item) commit() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3d28cf4..d97bf32 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -30,16 +30,21 @@ def test_fundamentals(self): cash_flow = fundamentals.get_sheet('913261697', 3) search_results = fundamentals.search('Adidas') - assert search_results and 'tickerId' in search_results[0] + self.assertIsNotNone(search_results) + self.assertIn('tickerId', search_results[0]) ads_id = search_results[0]['tickerId'] income_2 = fundamentals.get_income_facts(ads_id) income_analyse = fundamentals.get_income_analysis(ads_id) recommendation = fundamentals.get_recommendation(ads_id) ticker = fundamentals.get_ticker_ids(TestCrawler.SYMBOLS) - assert len(ticker) == 5 - assert any([symbol in ticker for symbol in TestCrawler.SYMBOLS]) - assert any([income, balance, cash_flow, brief, income_2, - income_analyse, recommendation, ticker]) + self.assertEqual(len(ticker), 5) + self.assertTrue( + any([symbol in ticker for symbol in TestCrawler.SYMBOLS]) + ) + self.assertTrue( + any([income, balance, cash_flow, brief, income_2, + income_analyse, recommendation, ticker]) + ) def test_crawler(self): """ @@ -48,12 +53,14 @@ def test_crawler(self): """ crawler = DataCrawler(True) price = crawler.get_last_price('ADS.F') - assert price + self.assertIsNotNone(price) price_stack = crawler.get_last_price_stack(TestCrawler.SYMBOLS_Y) - assert any([symbol in price_stack and price_stack[symbol] - for symbol in TestCrawler.SYMBOLS_Y]) + self.assertTrue( + any([symbol in price_stack and price_stack[symbol] + for symbol in TestCrawler.SYMBOLS_Y]) + ) ads_day = crawler.get_series('ADS.F', '1d') - assert ads_day + self.assertIsNotNone(ads_day) if __name__ == "__main__": diff --git a/tests/test_db.py b/tests/test_db.py index 45974ea..b623ba8 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -50,10 +50,10 @@ def test_create(self): create_database('database_create.sqlite') with db_session: stocks = Stock.select().count() - assert stocks == 30 + self.assertEqual(stocks, 30) prices = list(select(max(p.date) for p in Price)) - assert len(prices) == 1 - assert prices[0].strftime('%Y-%m-%d') == '2019-01-14' + self.assertEqual(len(prices), 1) + self.assertEqual(prices[0].strftime('%Y-%m-%d'), '2019-01-14') def test_update(self): """ @@ -75,17 +75,18 @@ def test_update(self): update.build() with db_session: prices = list(select(max(p.date) for p in Price)) - assert len(prices) == 1 - assert prices[0] > datetime.datetime.strptime('2019-01-14', - '%Y-%m-%d') + self.assertEqual(len(prices), 1) + my_date = datetime.datetime.strptime('2019-01-14', + '%Y-%m-%d') + self.assertGreater(prices[0], my_date) price_ctx = Price.select().count() data_ctx = Data.select().count() update.build() with db_session: price_ctx_now = Price.select().count() data_ctx_now = Data.select().count() - assert price_ctx_now == price_ctx - assert data_ctx_now == data_ctx + self.assertEqual(price_ctx_now, price_ctx) + self.assertEqual(data_ctx_now, data_ctx) def test_sync(self): """Tests sync tool @@ -104,7 +105,7 @@ def test_sync(self): sync.build() with db_session: stocks = Stock.select().count() - assert stocks == 71 + self.assertEqual(stocks, 70) if __name__ == "__main__":