Fixed credentials

.travis.yml

@@ -8,10 +8,12 @@ script:
   - tox
 deploy:
   provider: pypi
+  skip_cleanup: true
+  distributions: "bdist_wheel"
   user:
-    secure: "1WDqOWBY6tVEDRElNWLEws22r4GrHIn8pclYv2ivKJps4JzN3kREbuW/11oW6Cz8UOtMhUr8RwMtbupq73He9lP0y6AYrd4dr4EAW+G4woblMGo5k2L/m8Yt13RQ/jwP83+XNmdaGmZaP6TqN1/zYL9q2wAImoX0IqNu2csFsAMkVriqeuhy8IRra8YAChgHXHdCFFeZpJcWcZb45ej1sc9MbGgb+55+wVvA/gMzvkBLAEqC/P/Tm5FFvK5WW5vSHVrHMCeZvvh8m8v7vAo2Swx4ITzUvKyE5XdiKDfMrEAK0cW7iz1ghiP//2oKS/JaUNHOSYW/0sebn8EUBdyUSEqt/rN1l1+OietXxK6g3XK+vI/PKWF3ImcxpP0jM8laysLPfhrE6iQvAGJGgJObbpNeCXD7N0bsY5Rtpx8g1obARWLo2LwBEbmWKxtFi6goL2xHdLaGGp/og1bgSZv8h4S/raIOK62Ocu3D7Ju9ArrtCatXJ0sIhK5Nj7CFFEcjU8d+hMybPwspumi7LmULa97dPnRMzc2uJJhUAlINfKgSLXfrxtaThRJjQQKo0J0xcDEFroOIvcjoKQrUADwXyWGNwP6pT5SCDhnGGPnE8RjsSXAk1Qqj6Yl0GcdMwrCJiEp5qSEy9mvXic52jE2gOWdB8beDr2yOOBPdhQgrx1c="
+    secure: "dndHOzbxxOQ/9hBo5oRRxYEXhiQqdV4sKJOhWaugUm9N6GzULS0cPcTohQ2JjEsMnCkZBZlkwcKS3NB4sq+CB3wUi1RXIZKz0nH0J0fmTAFi4VFaxjsD2MffSXSWSWJmb6P5soG6G76RRlbSAwsqoRf/Me3qaHH3JJxMFGyHAe/TX49vGFBlDpzTApQ5O9v+7Y5jZlU/Rdb6X120WLW9Ah6PW8rLz7d7WjwBZTC7aCrd6c+w4M5pIVHc18zURXWBO4h6vat6DOhxxxLKSNPpLhmbbol1vsIT53Zj1XPKWkCarzHMbn6/4xxUpZewsE6us8Mryk2hopY8CPgAmARpxQQDTnxDe53OLrx1L+M7vsmlsuCmTofh6GsK3xF7IzDkCWi5/764Xyw/qKnZQ54sFwqYh8DSFl9ghiXw74tWDSnxFEgaQWMG7tzb5CRKx2k9eGnHiGgbzhjbHdBY5hg36xNA7tdaINpnLDPUUKL7PHJ4dclMMSOwvoPBzdEnPJa4FZsU6wnNbY7AC2p0OJg91HFAAXY4/3bsvGoDlFX00WXZdX3v1km+xDFzF87YEa876U+M0XshjZw9Fo5eP4s4H54lC0wR0pF42FDWNACg3LOA82Qk3XpZcao/4bmC+AZaK0/3h4iXTaE+Mb0sYKx9G6xTj01CIDUpF3LDrBoMaYY="
   password:
-    secure: "bobH2YMEIwCwFo5+XK2Y4wtI48xBH4q6UGIbD5kyxZ8ttUoPnNvKXGVofFNwwP7kO1312q5RC3HHYj0S0E1BShyt7G+/W/fPrIkAhu7U6XMQhN33mQH3lDqfXfqoH8kd3t2BoVNTbwaooI0icUD/LElH+r8pYBug/qI4NNL5u3S9gwQKh77u3P1sUNEju14DdjmA+9cTcfTQJtSLukV9kiHLpkwkcpM8AHX5HYhU+7+cHShNwwOpX0IrZre9sN0NfcBL8U4qE+wisPLvBJ7wjvnRakxS3A1JdnF96ncxyd/A+60IFLfdpKpmAOJVsnC9Q/NWnRpI6JqtvwIZuKPVBMZqsSft6a/ZMz1wfvfW5NrDZSSs4cSX29zlGBK7BVqwF3uO73BxEj4m+5HYnJULEHv5kF34XZ/lg1dHAU09nbWqM3vmozj2ITbeXrxRBx7WNFG/sFiIJgvJJPAAX/lXoKrZFWbnWK5NoN0hLI7dZJrUHlj/xcIg8WOLAYYjrSXXwFqrpLTLnFIKEwS+9ngSodKb5Tcwb6hQmwXbSL8Qz8QCw5HlN0K5BhRHMhdnZG9GqGzJeSsk84NBQWKfaxlW2a+0Julz6tGCejQMFw+p3HC9EGUPPp1j/cAhZlKpUZBJ4NjimL/UvselFWWDjmUrAVwbXB0hJbvfjGOBnDP/XEY="
+    secure: "C0KD/7KNq1mM7q8U+KaPf7zEzTVI9QOjBqVZ1+/uKHUOIRciQGo4vA1e9aAtC+MxXzEpxX4aUb4lHrxEx17Dqqcf4PNt5A9bz8mAf+8yO6Q3SjlxGcCAqS8JMTBWNXWaIHiGnQ3aPGA5rOq66d+mRZo5rRPl/Of+rKcauDK42vBR8XaE7W9N/5+Z9+v4WUjTuXR3LMoE1cx2QwHPr2lyMEhYkFlj1T9/s5O0rTWNKlzFih76AOsdnKsjM5cvrrulXujXaYLU89UakIuBd/Om5Qf2V4p5SN8gPaLzMfHoPQdWtJG6FE2uJOCDRv8tByyooF0ADD0PaO2tV5luS+wM4nMcPQI/ZUenMrFefHvk9bvNihyVeyIe7gFxPKqWJkVJeaWid6NT6IiZdRKsn0d+Wo0QLTuYq/qG9UoMQr9z8hqeESyPMLY5wzJugyYxZ2aP7HDdzV6BTy6W/5XhmL1yAUY+0xwi7ukwbXMB9Vs3IWWh4zHkuYYGzqhD7qosTEFOj4ZpcvHYR05EW8CDT7iwzwunZMMnNBRbXu5zQPhJdC2ndAh5Pgq8jVsvguUEgF8YyYF+Gtfm3bXVLl51Ia1XSlu6dpGoRd2wZgR0D6pFwnSzybwdV8zccsoW8vFggGioJgQrg0SmIUA8fckPwj5qFQ/9dAs2pIymqeI9Vv6EhE8="
   on:
     branch: master
 after_success:

MANIFEST.in

@@ -0,0 +1 @@
+include README.md LICENSE requirements.txt

README.md

@@ -1,11 +1,25 @@
+# pystockdb
+
 [![Build Status](https://travis-ci.org/portfolioplus/pystockdb.svg?branch=master)](https://travis-ci.org/portfolioplus/pystockdb)
 [![Coverage Status](https://coveralls.io/repos/github/portfolioplus/pystockdb/badge.svg?branch=master)](https://coveralls.io/github/portfolioplus/pystockdb?branch=master)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/07e6231a5a8c415a9f27736e02a286da)](https://www.codacy.com/app/SlashGordon/pystockdb?utm_source=github.com&utm_medium=referral&utm_content=portfolioplus/pystockdb&utm_campaign=Badge_Grade)
 
-# pystockdb
+Database for stocks based on [pony.orm](https://github.com/ponyorm/pony).
+This package provides an create, sync and update tool.
 
-database for stocks based on pony.orm:
+At the moment we are only support a few stocks.
+If you want to have more, please contribute [pytickersymbols](https://github.com/portfolioplus/pytickersymbols).
 
-# quick start
+## install
+
+```shell
+pip install pystockdb
+```
+
+## quick start
+
+In all samples we use sqlite but you are free to use other providers.
+For more information's please read [Connecting to the Database](https://docs.ponyorm.org/database.html).
 
 Install sqlite stock db:
 
@@ -20,13 +34,55 @@ config = {
     'currency': 'EUR',
     'db_args': {
         'provider': 'sqlite',
-        'filename': db_name,
+        'filename': 'demo.sqlite',
         'create_db': True
     },
 }
 create = CreateAndFillDataBase(config, logger)
 create.build()
 ```
-# issue tracker
+
+Update sqlite stock db:
+
+```python
+import logging
+from pystockdb.tools.update import UpdateDataBaseStocks
+
+logger = logging.getLogger('test')
+config = {
+    'symbols': ['ALL'],
+    'prices': True,       # update prices
+    'fundamentals': True, # update fundamental stock data
+    'db_args': {
+        'provider': 'sqlite',
+        'filename': 'demo.sqlite',
+        'create_db': False
+    },
+}
+update = UpdateDataBaseStocks(config, logger)
+update.build()
+```
+
+Sync sqlite stock db:
+
+```python
+import logging
+from pystockdb.tools.sync import SyncDataBaseStocks
+
+logger = logging.getLogger('test')
+config = {
+    'max_history': 1,
+    'indices': ['CAC 40'], # add new index to existing database
+    'currency': 'EUR',
+    'db_args': {
+        'provider': 'sqlite',
+        'filename': 'demo.sqlite',
+    },
+}
+sync = SyncDataBaseStocks(config, logger)
+sync.build()
+```
+
+## issue tracker
 
 [https://github.com/portfolioplus/pystockdb/issuese](https://github.com/portfolioplus/pystockdb/issues")

requirements.txt

@@ -1,4 +1,5 @@
-pytickersymbols==1.0.8
-yfinance==0.1.43
+pytickersymbols==1.1.3
+pandas==0.24.2
+yfinance==0.1.44
 uplink==0.9.0
-pony==0.7.10 
+pony==0.7.10

setup.py

@@ -4,44 +4,43 @@
 
  Copyright 2019 Slash Gordon
 
- Use of this source code is governed by a GNU General Public License v3 or 
+ Use of this source code is governed by a GNU General Public License v3 or
  later that can be
  found in the LICENSE file.
 """
 
 from setuptools import setup, find_packages
 
 EXCLUDE_FROM_PACKAGES = ['test', 'test.*', 'test*']
-VERSION = '0.0.1'
-
-
-def get_requirements(requirements):
-    with open(requirements) as requirement_file:
-        content = requirement_file.readlines()
-    content = [x.strip() for x in content]
-    return content
-
+VERSION = '1.0.0'
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+INSTALL_REQUIRES = (
+    [
+        'pytickersymbols>=1.1.3', 'pandas==0.24.2', 'yfinance>=0.1.44',
+        'uplink>=0.9.0', 'pony==0.7.10'
+    ]
+)
+
 setup(
     name="pystockdb",
     version=VERSION,
     author="Slash Gordon",
-    author_email="slash.gordon.dev@gmail.com ",
-    py_modules=['pystockdb'],
+    author_email="slash.gordon.dev@gmail.com",
     package_dir={'': 'src'},
     description="Simple stock db with tools.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/portfolioplus/pystockdb",
+    install_requires=INSTALL_REQUIRES,
     packages=find_packages('src', exclude=EXCLUDE_FROM_PACKAGES),
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'Natural Language :: English',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
+        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
@@ -53,4 +52,4 @@ def get_requirements(requirements):
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Topic :: Office/Business :: Financial :: Investment',
     ],
-)
+)

src/pystockdb/db/schema/stocks.py

@@ -163,9 +163,9 @@ class Argument(db.Entity):
 
 class Data(db.Entity):
     id = PrimaryKey(int, auto=True)
-    date = Required(datetime, default=lambda: datetime.now())
+    date = Required(datetime, default=datetime.now())
     data = Required(Json)
-    md5 = Required(str)
+    hash = Required(str)
     data_item = Required('DataItem')
 
 

src/pystockdb/tools/base.py

@@ -11,6 +11,8 @@
 
 from pony.orm import commit, db_session
 
+from pytickersymbols import PyTickerSymbols
+
 from pystockdb.db.schema.stocks import (Index, Item, PriceItem, Stock, Tag,
                                         Type, db)
 from pystockdb.tools.data_crawler import DataCrawler
@@ -25,11 +27,12 @@ def __init__(self, arguments: dict, logger: logging.Logger):
         self.logger = logger
         self.db_args = arguments["db_args"]
         self.arguments = arguments
+        self.ticker_symbols = PyTickerSymbols()
         # has connection
         if db.provider is None:
             # prepare db
             db.bind(**self.db_args)
-            if self.arguments.get('create', False):
+            if self.db_args.get('create_db', False):
                 db.generate_mapping(check_tables=False)
                 db.drop_all_tables(with_all_data=True)
                 db.create_tables()
@@ -75,7 +78,6 @@ def __add_stock_to_index(self, index, stock_info):
             stock = Stock(name=stock_info['name'],
                           price_item=PriceItem(item=Item()))
             # add symbols
-            "".startswith
             yao = Tag.get(name=Tag.YAO)
             gog = Tag.get(name=Tag.GOG)
             usd = Tag.get(name=Tag.USD)
@@ -117,6 +119,10 @@ def download_historicals(self, symbols, start, end):
             ids = [symbol.name for symbol in chunk]
             series = crawler.get_series_stack(ids, start=start, end=end)
             for symbol in chunk:
+                self.logger.debug(
+                    'Add prices for {} from {} until {}.'.format(symbol.name,
+                                                                 start, end)
+                )
                 for value in series[symbol.name]:
                     symbol.prices.create(**value)
             commit()

src/pystockdb/tools/create.py

@@ -11,10 +11,7 @@
 import logging
 from datetime import timedelta
 
-from pony.orm import commit, db_session
-from pytickersymbols import PyTickerSymbols
-
-from pystockdb.db.schema.stocks import Symbol, Tag, Type
+from pystockdb.db.schema.stocks import Symbol, Tag
 from pystockdb.tools.base import DBBase
 
 
@@ -29,7 +26,6 @@ def __init__(self, arguments: dict, logger: logging.Logger):
         self.currency = arguments['currency']
         self.history = arguments['max_history']
         self.indices_list = arguments['indices']
-        self.ticker_symbols = PyTickerSymbols()
 
     def build(self):
         """

src/pystockdb/tools/fundamentals.py

@@ -121,12 +121,12 @@ def get_income(self, ticker_id):
 
     @returns.json
     @get('/api/securities/stock/{ticker_id}/statementsV2Detail')
-    def get_sheet(self, ticker_id, type: Query,
+    def get_sheet(self, ticker_id, sheet_type: Query("type"),
                   query_number: Query('queryNumber') = 300):
         """
         Returns financial sheet(income, balance, cash flow) data for stock
         :param ticker_id: ticker id
-        :param type: income=1, balance=2 and cashflow = 3
+        :param sheet_type: income=1, balance=2 and cashflow = 3
         :param queryNumber:
         :return:
         """

src/pystockdb/tools/update.py

@@ -68,7 +68,7 @@ def update_prices(self):
         for key in update:
             start = datetime.datetime.strptime(key, '%Y-%m-%d')
             end = datetime.datetime.now()
-            if start >= end:
+            if start.date() >= end.date():
                 continue
             self.download_historicals(update[key],
                                       start=start.strftime('%Y-%m-%d'),
@@ -89,7 +89,16 @@ def update_fundamentals(self):
         fundamentals = Fundamentals(base_url=Fundamentals.BASE_URL)
         tickers = fundamentals.get_ticker_ids(gog_syms)
         for ticker in tickers:
-            stock = [sto[0] for sto in stocks if sto[1] == ticker][0]
+            self.logger.debug(
+                'Download fundamentals for {}'.format(tickers[ticker])
+            )
+            stock = [sto[0] for sto in stocks if sto[1] == ticker]
+            if len(stock) != 1:
+                self.logger.warning(
+                    'Can not download fundamentals for {}'.format(ticker)
+                    )
+                continue
+            stock = stock[0]
             ica = fundamentals.get_income_analysis(tickers[ticker])
             ifc = fundamentals.get_income_facts(tickers[ticker])
             rec = fundamentals.get_recommendation(tickers[ticker])
@@ -99,16 +108,16 @@ def update_fundamentals(self):
             for val in [(ica, Tag.ICA), (ifc, Tag.ICF), (rec, Tag.REC),
                         (ble, Tag.BLE), (ico, Tag.ICO), (csh, Tag.CSH)]:
                 # hash stock name, tag and data
-                m = hashlib.md5()
+                m = hashlib.sha256()
                 m.update(val[1].encode('UTF-8'))
                 m.update(stock.name.encode('UTF-8'))
                 data_str = json.dumps(val[0])
                 m.update(data_str.encode('UTF-8'))
-                md5 = m.hexdigest()
+                shahash = m.hexdigest()
                 # only add data if not exist
-                if Data.get(md5=md5) is None:
+                if Data.get(hash=shahash) is None:
                     tag = Tag.get(name=val[1])
-                    obj = Data(data=val[0], md5=md5,
+                    obj = Data(data=val[0], hash=shahash,
                                data_item=DataItem(item=Item(tags=[tag])))
                     stock.data_items.add(obj.data_item)
         commit()

tests/test_crawler.py

@@ -30,16 +30,21 @@ def test_fundamentals(self):
         cash_flow = fundamentals.get_sheet('913261697', 3)
 
         search_results = fundamentals.search('Adidas')
-        assert search_results and 'tickerId' in search_results[0]
+        self.assertIsNotNone(search_results)
+        self.assertIn('tickerId', search_results[0])
         ads_id = search_results[0]['tickerId']
         income_2 = fundamentals.get_income_facts(ads_id)
         income_analyse = fundamentals.get_income_analysis(ads_id)
         recommendation = fundamentals.get_recommendation(ads_id)
         ticker = fundamentals.get_ticker_ids(TestCrawler.SYMBOLS)
-        assert len(ticker) == 5
-        assert any([symbol in ticker for symbol in TestCrawler.SYMBOLS])
-        assert any([income, balance, cash_flow, brief, income_2,
-                   income_analyse, recommendation, ticker])
+        self.assertEqual(len(ticker), 5)
+        self.assertTrue(
+          any([symbol in ticker for symbol in TestCrawler.SYMBOLS])
+        )
+        self.assertTrue(
+          any([income, balance, cash_flow, brief, income_2,
+              income_analyse, recommendation, ticker])
+        )
 
     def test_crawler(self):
         """
@@ -48,12 +53,14 @@ def test_crawler(self):
         """
         crawler = DataCrawler(True)
         price = crawler.get_last_price('ADS.F')
-        assert price
+        self.assertIsNotNone(price)
         price_stack = crawler.get_last_price_stack(TestCrawler.SYMBOLS_Y)
-        assert any([symbol in price_stack and price_stack[symbol]
-                    for symbol in TestCrawler.SYMBOLS_Y])
+        self.assertTrue(
+          any([symbol in price_stack and price_stack[symbol]
+              for symbol in TestCrawler.SYMBOLS_Y])
+        )
         ads_day = crawler.get_series('ADS.F', '1d')
-        assert ads_day
+        self.assertIsNotNone(ads_day)
 
 
 if __name__ == "__main__":