From 68cbda196357399c83b8549096cda4a231b257de Mon Sep 17 00:00:00 2001 From: Slash Gordon Date: Fri, 2 Aug 2019 19:33:30 +0200 Subject: [PATCH] Fixed credentials --- .travis.yml | 6 ++- MANIFEST.in | 1 + README.md | 66 ++++++++++++++++++++++++++--- requirements.txt | 7 +-- setup.py | 27 ++++++------ src/pystockdb/db/schema/stocks.py | 4 +- src/pystockdb/tools/base.py | 10 ++++- src/pystockdb/tools/create.py | 6 +-- src/pystockdb/tools/fundamentals.py | 4 +- src/pystockdb/tools/update.py | 21 ++++++--- tests/test_crawler.py | 25 +++++++---- tests/test_db.py | 19 +++++---- tox.ini | 2 +- 13 files changed, 138 insertions(+), 60 deletions(-) create mode 100644 MANIFEST.in diff --git a/.travis.yml b/.travis.yml index 73a5e0f..59ec834 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,12 @@ script: - tox deploy: provider: pypi + skip_cleanup: true + distributions: "bdist_wheel" user: - secure: "1WDqOWBY6tVEDRElNWLEws22r4GrHIn8pclYv2ivKJps4JzN3kREbuW/11oW6Cz8UOtMhUr8RwMtbupq73He9lP0y6AYrd4dr4EAW+G4woblMGo5k2L/m8Yt13RQ/jwP83+XNmdaGmZaP6TqN1/zYL9q2wAImoX0IqNu2csFsAMkVriqeuhy8IRra8YAChgHXHdCFFeZpJcWcZb45ej1sc9MbGgb+55+wVvA/gMzvkBLAEqC/P/Tm5FFvK5WW5vSHVrHMCeZvvh8m8v7vAo2Swx4ITzUvKyE5XdiKDfMrEAK0cW7iz1ghiP//2oKS/JaUNHOSYW/0sebn8EUBdyUSEqt/rN1l1+OietXxK6g3XK+vI/PKWF3ImcxpP0jM8laysLPfhrE6iQvAGJGgJObbpNeCXD7N0bsY5Rtpx8g1obARWLo2LwBEbmWKxtFi6goL2xHdLaGGp/og1bgSZv8h4S/raIOK62Ocu3D7Ju9ArrtCatXJ0sIhK5Nj7CFFEcjU8d+hMybPwspumi7LmULa97dPnRMzc2uJJhUAlINfKgSLXfrxtaThRJjQQKo0J0xcDEFroOIvcjoKQrUADwXyWGNwP6pT5SCDhnGGPnE8RjsSXAk1Qqj6Yl0GcdMwrCJiEp5qSEy9mvXic52jE2gOWdB8beDr2yOOBPdhQgrx1c=" + secure: "dndHOzbxxOQ/9hBo5oRRxYEXhiQqdV4sKJOhWaugUm9N6GzULS0cPcTohQ2JjEsMnCkZBZlkwcKS3NB4sq+CB3wUi1RXIZKz0nH0J0fmTAFi4VFaxjsD2MffSXSWSWJmb6P5soG6G76RRlbSAwsqoRf/Me3qaHH3JJxMFGyHAe/TX49vGFBlDpzTApQ5O9v+7Y5jZlU/Rdb6X120WLW9Ah6PW8rLz7d7WjwBZTC7aCrd6c+w4M5pIVHc18zURXWBO4h6vat6DOhxxxLKSNPpLhmbbol1vsIT53Zj1XPKWkCarzHMbn6/4xxUpZewsE6us8Mryk2hopY8CPgAmARpxQQDTnxDe53OLrx1L+M7vsmlsuCmTofh6GsK3xF7IzDkCWi5/764Xyw/qKnZQ54sFwqYh8DSFl9ghiXw74tWDSnxFEgaQWMG7tzb5CRKx2k9eGnHiGgbzhjbHdBY5hg36xNA7tdaINpnLDPUUKL7PHJ4dclMMSOwvoPBzdEnPJa4FZsU6wnNbY7AC2p0OJg91HFAAXY4/3bsvGoDlFX00WXZdX3v1km+xDFzF87YEa876U+M0XshjZw9Fo5eP4s4H54lC0wR0pF42FDWNACg3LOA82Qk3XpZcao/4bmC+AZaK0/3h4iXTaE+Mb0sYKx9G6xTj01CIDUpF3LDrBoMaYY=" password: - secure: "bobH2YMEIwCwFo5+XK2Y4wtI48xBH4q6UGIbD5kyxZ8ttUoPnNvKXGVofFNwwP7kO1312q5RC3HHYj0S0E1BShyt7G+/W/fPrIkAhu7U6XMQhN33mQH3lDqfXfqoH8kd3t2BoVNTbwaooI0icUD/LElH+r8pYBug/qI4NNL5u3S9gwQKh77u3P1sUNEju14DdjmA+9cTcfTQJtSLukV9kiHLpkwkcpM8AHX5HYhU+7+cHShNwwOpX0IrZre9sN0NfcBL8U4qE+wisPLvBJ7wjvnRakxS3A1JdnF96ncxyd/A+60IFLfdpKpmAOJVsnC9Q/NWnRpI6JqtvwIZuKPVBMZqsSft6a/ZMz1wfvfW5NrDZSSs4cSX29zlGBK7BVqwF3uO73BxEj4m+5HYnJULEHv5kF34XZ/lg1dHAU09nbWqM3vmozj2ITbeXrxRBx7WNFG/sFiIJgvJJPAAX/lXoKrZFWbnWK5NoN0hLI7dZJrUHlj/xcIg8WOLAYYjrSXXwFqrpLTLnFIKEwS+9ngSodKb5Tcwb6hQmwXbSL8Qz8QCw5HlN0K5BhRHMhdnZG9GqGzJeSsk84NBQWKfaxlW2a+0Julz6tGCejQMFw+p3HC9EGUPPp1j/cAhZlKpUZBJ4NjimL/UvselFWWDjmUrAVwbXB0hJbvfjGOBnDP/XEY=" + secure: "C0KD/7KNq1mM7q8U+KaPf7zEzTVI9QOjBqVZ1+/uKHUOIRciQGo4vA1e9aAtC+MxXzEpxX4aUb4lHrxEx17Dqqcf4PNt5A9bz8mAf+8yO6Q3SjlxGcCAqS8JMTBWNXWaIHiGnQ3aPGA5rOq66d+mRZo5rRPl/Of+rKcauDK42vBR8XaE7W9N/5+Z9+v4WUjTuXR3LMoE1cx2QwHPr2lyMEhYkFlj1T9/s5O0rTWNKlzFih76AOsdnKsjM5cvrrulXujXaYLU89UakIuBd/Om5Qf2V4p5SN8gPaLzMfHoPQdWtJG6FE2uJOCDRv8tByyooF0ADD0PaO2tV5luS+wM4nMcPQI/ZUenMrFefHvk9bvNihyVeyIe7gFxPKqWJkVJeaWid6NT6IiZdRKsn0d+Wo0QLTuYq/qG9UoMQr9z8hqeESyPMLY5wzJugyYxZ2aP7HDdzV6BTy6W/5XhmL1yAUY+0xwi7ukwbXMB9Vs3IWWh4zHkuYYGzqhD7qosTEFOj4ZpcvHYR05EW8CDT7iwzwunZMMnNBRbXu5zQPhJdC2ndAh5Pgq8jVsvguUEgF8YyYF+Gtfm3bXVLl51Ia1XSlu6dpGoRd2wZgR0D6pFwnSzybwdV8zccsoW8vFggGioJgQrg0SmIUA8fckPwj5qFQ/9dAs2pIymqeI9Vv6EhE8=" on: branch: master after_success: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d18a98d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.md LICENSE requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index d704f34..58da2a9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,25 @@ +# pystockdb + [![Build Status](https://travis-ci.org/portfolioplus/pystockdb.svg?branch=master)](https://travis-ci.org/portfolioplus/pystockdb) [![Coverage Status](https://coveralls.io/repos/github/portfolioplus/pystockdb/badge.svg?branch=master)](https://coveralls.io/github/portfolioplus/pystockdb?branch=master) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/07e6231a5a8c415a9f27736e02a286da)](https://www.codacy.com/app/SlashGordon/pystockdb?utm_source=github.com&utm_medium=referral&utm_content=portfolioplus/pystockdb&utm_campaign=Badge_Grade) -# pystockdb +Database for stocks based on [pony.orm](https://github.com/ponyorm/pony). +This package provides an create, sync and update tool. -database for stocks based on pony.orm: +At the moment we are only support a few stocks. +If you want to have more, please contribute [pytickersymbols](https://github.com/portfolioplus/pytickersymbols). -# quick start +## install + +```shell +pip install pystockdb +``` + +## quick start + +In all samples we use sqlite but you are free to use other providers. +For more information's please read [Connecting to the Database](https://docs.ponyorm.org/database.html). Install sqlite stock db: @@ -20,13 +34,55 @@ config = { 'currency': 'EUR', 'db_args': { 'provider': 'sqlite', - 'filename': db_name, + 'filename': 'demo.sqlite', 'create_db': True }, } create = CreateAndFillDataBase(config, logger) create.build() ``` -# issue tracker + +Update sqlite stock db: + +```python +import logging +from pystockdb.tools.update import UpdateDataBaseStocks + +logger = logging.getLogger('test') +config = { + 'symbols': ['ALL'], + 'prices': True, # update prices + 'fundamentals': True, # update fundamental stock data + 'db_args': { + 'provider': 'sqlite', + 'filename': 'demo.sqlite', + 'create_db': False + }, +} +update = UpdateDataBaseStocks(config, logger) +update.build() +``` + +Sync sqlite stock db: + +```python +import logging +from pystockdb.tools.sync import SyncDataBaseStocks + +logger = logging.getLogger('test') +config = { + 'max_history': 1, + 'indices': ['CAC 40'], # add new index to existing database + 'currency': 'EUR', + 'db_args': { + 'provider': 'sqlite', + 'filename': 'demo.sqlite', + }, +} +sync = SyncDataBaseStocks(config, logger) +sync.build() +``` + +## issue tracker [https://github.com/portfolioplus/pystockdb/issuese](https://github.com/portfolioplus/pystockdb/issues") diff --git a/requirements.txt b/requirements.txt index 7b7406f..212df31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -pytickersymbols==1.0.8 -yfinance==0.1.43 +pytickersymbols==1.1.3 +pandas==0.24.2 +yfinance==0.1.44 uplink==0.9.0 -pony==0.7.10 \ No newline at end of file +pony==0.7.10 diff --git a/setup.py b/setup.py index c570894..e3d47fd 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ Copyright 2019 Slash Gordon - Use of this source code is governed by a GNU General Public License v3 or + Use of this source code is governed by a GNU General Public License v3 or later that can be found in the LICENSE file. """ @@ -12,36 +12,35 @@ from setuptools import setup, find_packages EXCLUDE_FROM_PACKAGES = ['test', 'test.*', 'test*'] -VERSION = '0.0.1' - - -def get_requirements(requirements): - with open(requirements) as requirement_file: - content = requirement_file.readlines() - content = [x.strip() for x in content] - return content - +VERSION = '1.0.0' with open("README.md", "r") as fh: long_description = fh.read() +INSTALL_REQUIRES = ( + [ + 'pytickersymbols>=1.1.3', 'pandas==0.24.2', 'yfinance>=0.1.44', + 'uplink>=0.9.0', 'pony==0.7.10' + ] +) + setup( name="pystockdb", version=VERSION, author="Slash Gordon", - author_email="slash.gordon.dev@gmail.com ", - py_modules=['pystockdb'], + author_email="slash.gordon.dev@gmail.com", package_dir={'': 'src'}, description="Simple stock db with tools.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/portfolioplus/pystockdb", + install_requires=INSTALL_REQUIRES, packages=find_packages('src', exclude=EXCLUDE_FROM_PACKAGES), classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Natural Language :: English', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', @@ -53,4 +52,4 @@ def get_requirements(requirements): 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Office/Business :: Financial :: Investment', ], -) \ No newline at end of file +) diff --git a/src/pystockdb/db/schema/stocks.py b/src/pystockdb/db/schema/stocks.py index 1f65ac8..aca05ad 100644 --- a/src/pystockdb/db/schema/stocks.py +++ b/src/pystockdb/db/schema/stocks.py @@ -163,9 +163,9 @@ class Argument(db.Entity): class Data(db.Entity): id = PrimaryKey(int, auto=True) - date = Required(datetime, default=lambda: datetime.now()) + date = Required(datetime, default=datetime.now()) data = Required(Json) - md5 = Required(str) + hash = Required(str) data_item = Required('DataItem') diff --git a/src/pystockdb/tools/base.py b/src/pystockdb/tools/base.py index 11dcbcb..6598a7d 100644 --- a/src/pystockdb/tools/base.py +++ b/src/pystockdb/tools/base.py @@ -11,6 +11,8 @@ from pony.orm import commit, db_session +from pytickersymbols import PyTickerSymbols + from pystockdb.db.schema.stocks import (Index, Item, PriceItem, Stock, Tag, Type, db) from pystockdb.tools.data_crawler import DataCrawler @@ -25,11 +27,12 @@ def __init__(self, arguments: dict, logger: logging.Logger): self.logger = logger self.db_args = arguments["db_args"] self.arguments = arguments + self.ticker_symbols = PyTickerSymbols() # has connection if db.provider is None: # prepare db db.bind(**self.db_args) - if self.arguments.get('create', False): + if self.db_args.get('create_db', False): db.generate_mapping(check_tables=False) db.drop_all_tables(with_all_data=True) db.create_tables() @@ -75,7 +78,6 @@ def __add_stock_to_index(self, index, stock_info): stock = Stock(name=stock_info['name'], price_item=PriceItem(item=Item())) # add symbols - "".startswith yao = Tag.get(name=Tag.YAO) gog = Tag.get(name=Tag.GOG) usd = Tag.get(name=Tag.USD) @@ -117,6 +119,10 @@ def download_historicals(self, symbols, start, end): ids = [symbol.name for symbol in chunk] series = crawler.get_series_stack(ids, start=start, end=end) for symbol in chunk: + self.logger.debug( + 'Add prices for {} from {} until {}.'.format(symbol.name, + start, end) + ) for value in series[symbol.name]: symbol.prices.create(**value) commit() diff --git a/src/pystockdb/tools/create.py b/src/pystockdb/tools/create.py index e36a002..787416c 100644 --- a/src/pystockdb/tools/create.py +++ b/src/pystockdb/tools/create.py @@ -11,10 +11,7 @@ import logging from datetime import timedelta -from pony.orm import commit, db_session -from pytickersymbols import PyTickerSymbols - -from pystockdb.db.schema.stocks import Symbol, Tag, Type +from pystockdb.db.schema.stocks import Symbol, Tag from pystockdb.tools.base import DBBase @@ -29,7 +26,6 @@ def __init__(self, arguments: dict, logger: logging.Logger): self.currency = arguments['currency'] self.history = arguments['max_history'] self.indices_list = arguments['indices'] - self.ticker_symbols = PyTickerSymbols() def build(self): """ diff --git a/src/pystockdb/tools/fundamentals.py b/src/pystockdb/tools/fundamentals.py index 82ae302..502f34e 100644 --- a/src/pystockdb/tools/fundamentals.py +++ b/src/pystockdb/tools/fundamentals.py @@ -121,12 +121,12 @@ def get_income(self, ticker_id): @returns.json @get('/api/securities/stock/{ticker_id}/statementsV2Detail') - def get_sheet(self, ticker_id, type: Query, + def get_sheet(self, ticker_id, sheet_type: Query("type"), query_number: Query('queryNumber') = 300): """ Returns financial sheet(income, balance, cash flow) data for stock :param ticker_id: ticker id - :param type: income=1, balance=2 and cashflow = 3 + :param sheet_type: income=1, balance=2 and cashflow = 3 :param queryNumber: :return: """ diff --git a/src/pystockdb/tools/update.py b/src/pystockdb/tools/update.py index 74fc7c9..3086054 100644 --- a/src/pystockdb/tools/update.py +++ b/src/pystockdb/tools/update.py @@ -68,7 +68,7 @@ def update_prices(self): for key in update: start = datetime.datetime.strptime(key, '%Y-%m-%d') end = datetime.datetime.now() - if start >= end: + if start.date() >= end.date(): continue self.download_historicals(update[key], start=start.strftime('%Y-%m-%d'), @@ -89,7 +89,16 @@ def update_fundamentals(self): fundamentals = Fundamentals(base_url=Fundamentals.BASE_URL) tickers = fundamentals.get_ticker_ids(gog_syms) for ticker in tickers: - stock = [sto[0] for sto in stocks if sto[1] == ticker][0] + self.logger.debug( + 'Download fundamentals for {}'.format(tickers[ticker]) + ) + stock = [sto[0] for sto in stocks if sto[1] == ticker] + if len(stock) != 1: + self.logger.warning( + 'Can not download fundamentals for {}'.format(ticker) + ) + continue + stock = stock[0] ica = fundamentals.get_income_analysis(tickers[ticker]) ifc = fundamentals.get_income_facts(tickers[ticker]) rec = fundamentals.get_recommendation(tickers[ticker]) @@ -99,16 +108,16 @@ def update_fundamentals(self): for val in [(ica, Tag.ICA), (ifc, Tag.ICF), (rec, Tag.REC), (ble, Tag.BLE), (ico, Tag.ICO), (csh, Tag.CSH)]: # hash stock name, tag and data - m = hashlib.md5() + m = hashlib.sha256() m.update(val[1].encode('UTF-8')) m.update(stock.name.encode('UTF-8')) data_str = json.dumps(val[0]) m.update(data_str.encode('UTF-8')) - md5 = m.hexdigest() + shahash = m.hexdigest() # only add data if not exist - if Data.get(md5=md5) is None: + if Data.get(hash=shahash) is None: tag = Tag.get(name=val[1]) - obj = Data(data=val[0], md5=md5, + obj = Data(data=val[0], hash=shahash, data_item=DataItem(item=Item(tags=[tag]))) stock.data_items.add(obj.data_item) commit() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3d28cf4..d97bf32 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -30,16 +30,21 @@ def test_fundamentals(self): cash_flow = fundamentals.get_sheet('913261697', 3) search_results = fundamentals.search('Adidas') - assert search_results and 'tickerId' in search_results[0] + self.assertIsNotNone(search_results) + self.assertIn('tickerId', search_results[0]) ads_id = search_results[0]['tickerId'] income_2 = fundamentals.get_income_facts(ads_id) income_analyse = fundamentals.get_income_analysis(ads_id) recommendation = fundamentals.get_recommendation(ads_id) ticker = fundamentals.get_ticker_ids(TestCrawler.SYMBOLS) - assert len(ticker) == 5 - assert any([symbol in ticker for symbol in TestCrawler.SYMBOLS]) - assert any([income, balance, cash_flow, brief, income_2, - income_analyse, recommendation, ticker]) + self.assertEqual(len(ticker), 5) + self.assertTrue( + any([symbol in ticker for symbol in TestCrawler.SYMBOLS]) + ) + self.assertTrue( + any([income, balance, cash_flow, brief, income_2, + income_analyse, recommendation, ticker]) + ) def test_crawler(self): """ @@ -48,12 +53,14 @@ def test_crawler(self): """ crawler = DataCrawler(True) price = crawler.get_last_price('ADS.F') - assert price + self.assertIsNotNone(price) price_stack = crawler.get_last_price_stack(TestCrawler.SYMBOLS_Y) - assert any([symbol in price_stack and price_stack[symbol] - for symbol in TestCrawler.SYMBOLS_Y]) + self.assertTrue( + any([symbol in price_stack and price_stack[symbol] + for symbol in TestCrawler.SYMBOLS_Y]) + ) ads_day = crawler.get_series('ADS.F', '1d') - assert ads_day + self.assertIsNotNone(ads_day) if __name__ == "__main__": diff --git a/tests/test_db.py b/tests/test_db.py index 45974ea..b623ba8 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -50,10 +50,10 @@ def test_create(self): create_database('database_create.sqlite') with db_session: stocks = Stock.select().count() - assert stocks == 30 + self.assertEqual(stocks, 30) prices = list(select(max(p.date) for p in Price)) - assert len(prices) == 1 - assert prices[0].strftime('%Y-%m-%d') == '2019-01-14' + self.assertEqual(len(prices), 1) + self.assertEqual(prices[0].strftime('%Y-%m-%d'), '2019-01-14') def test_update(self): """ @@ -75,17 +75,18 @@ def test_update(self): update.build() with db_session: prices = list(select(max(p.date) for p in Price)) - assert len(prices) == 1 - assert prices[0] > datetime.datetime.strptime('2019-01-14', - '%Y-%m-%d') + self.assertEqual(len(prices), 1) + my_date = datetime.datetime.strptime('2019-01-14', + '%Y-%m-%d') + self.assertGreater(prices[0], my_date) price_ctx = Price.select().count() data_ctx = Data.select().count() update.build() with db_session: price_ctx_now = Price.select().count() data_ctx_now = Data.select().count() - assert price_ctx_now == price_ctx - assert data_ctx_now == data_ctx + self.assertEqual(price_ctx_now, price_ctx) + self.assertEqual(data_ctx_now, data_ctx) def test_sync(self): """Tests sync tool @@ -104,7 +105,7 @@ def test_sync(self): sync.build() with db_session: stocks = Stock.select().count() - assert stocks == 71 + self.assertEqual(stocks, 70) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index 0fe43f2..932eb0e 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ basepython = py36: python3.6 commands = pytest tests/ --cov src/pystockdb --cov-report term-missing - python setup.py sdist bdist bdist_wheel + python setup.py bdist_wheel [testenv:flake8] max-line-length = 80