Fix build.

MANIFEST.in

@@ -0,0 +1 @@
+include README.md LICENSE requirements.txt

README.md

@@ -1,11 +1,25 @@
+# pystockdb
+
 [![Build Status](https://travis-ci.org/portfolioplus/pystockdb.svg?branch=master)](https://travis-ci.org/portfolioplus/pystockdb)
 [![Coverage Status](https://coveralls.io/repos/github/portfolioplus/pystockdb/badge.svg?branch=master)](https://coveralls.io/github/portfolioplus/pystockdb?branch=master)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/07e6231a5a8c415a9f27736e02a286da)](https://www.codacy.com/app/SlashGordon/pystockdb?utm_source=github.com&utm_medium=referral&utm_content=portfolioplus/pystockdb&utm_campaign=Badge_Grade)
 
-# pystockdb
+Database for stocks based on [pony.orm](https://github.com/ponyorm/pony).
+This package provides an create, sync and update tool.
 
-database for stocks based on pony.orm:
+At the moment we are only support a few stocks.
+If you want to have more, please contribute [pytickersymbols](https://github.com/portfolioplus/pytickersymbols).
 
-# quick start
+## install
+
+```shell
+pip install pystockdb
+```
+
+## quick start
+
+In all samples we use sqlite but you are free to use other providers.
+For more information's please read [Connecting to the Database](https://docs.ponyorm.org/database.html).
 
 Install sqlite stock db:
 
@@ -20,13 +34,55 @@ config = {
     'currency': 'EUR',
     'db_args': {
         'provider': 'sqlite',
-        'filename': db_name,
+        'filename': 'demo.sqlite',
         'create_db': True
     },
 }
 create = CreateAndFillDataBase(config, logger)
 create.build()
 ```
-# issue tracker
+
+Update sqlite stock db:
+
+```python
+import logging
+from pystockdb.tools.update import UpdateDataBaseStocks
+
+logger = logging.getLogger('test')
+config = {
+    'symbols': ['ALL'],
+    'prices': True,       # update prices
+    'fundamentals': True, # update fundamental stock data
+    'db_args': {
+        'provider': 'sqlite',
+        'filename': 'demo.sqlite',
+        'create_db': False
+    },
+}
+update = UpdateDataBaseStocks(config, logger)
+update.build()
+```
+
+Sync sqlite stock db:
+
+```python
+import logging
+from pystockdb.tools.sync import SyncDataBaseStocks
+
+logger = logging.getLogger('test')
+config = {
+    'max_history': 1,
+    'indices': ['CAC 40'], # add new index to existing database
+    'currency': 'EUR',
+    'db_args': {
+        'provider': 'sqlite',
+        'filename': 'demo.sqlite',
+    },
+}
+sync = SyncDataBaseStocks(config, logger)
+sync.build()
+```
+
+## issue tracker
 
 [https://github.com/portfolioplus/pystockdb/issuese](https://github.com/portfolioplus/pystockdb/issues")

requirements.txt

@@ -1,4 +1,5 @@
-pytickersymbols==1.0.8
-yfinance==0.1.43
+pytickersymbols==1.1.3
+pandas==0.24.2
+yfinance==0.1.44
 uplink==0.9.0
-pony==0.7.10 
+pony==0.7.10

setup.py

@@ -4,7 +4,7 @@
 
  Copyright 2019 Slash Gordon
 
- Use of this source code is governed by a GNU General Public License v3 or 
+ Use of this source code is governed by a GNU General Public License v3 or
  later that can be
  found in the LICENSE file.
 """
@@ -14,34 +14,33 @@
 EXCLUDE_FROM_PACKAGES = ['test', 'test.*', 'test*']
 VERSION = '0.0.1'
 
-
-def get_requirements(requirements):
-    with open(requirements) as requirement_file:
-        content = requirement_file.readlines()
-    content = [x.strip() for x in content]
-    return content
-
-
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
+INSTALL_REQUIRES = (
+    [
+        'pytickersymbols>=1.1.3', 'pandas==0.24.2', 'yfinance>=0.1.44',
+        'uplink>=0.9.0', 'pony==0.7.10'
+    ]
+)
+
 setup(
     name="pystockdb",
     version=VERSION,
     author="Slash Gordon",
-    author_email="slash.gordon.dev@gmail.com ",
-    py_modules=['pystockdb'],
+    author_email="slash.gordon.dev@gmail.com",
     package_dir={'': 'src'},
     description="Simple stock db with tools.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/portfolioplus/pystockdb",
+    install_requires=INSTALL_REQUIRES,
     packages=find_packages('src', exclude=EXCLUDE_FROM_PACKAGES),
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'Natural Language :: English',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
+        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
@@ -53,4 +52,4 @@ def get_requirements(requirements):
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Topic :: Office/Business :: Financial :: Investment',
     ],
-)
+)

src/pystockdb/db/schema/stocks.py

@@ -163,9 +163,9 @@ class Argument(db.Entity):
 
 class Data(db.Entity):
     id = PrimaryKey(int, auto=True)
-    date = Required(datetime, default=lambda: datetime.now())
+    date = Required(datetime, default=datetime.now())
     data = Required(Json)
-    md5 = Required(str)
+    hash = Required(str)
     data_item = Required('DataItem')
 
 

src/pystockdb/tools/base.py

@@ -11,6 +11,8 @@
 
 from pony.orm import commit, db_session
 
+from pytickersymbols import PyTickerSymbols
+
 from pystockdb.db.schema.stocks import (Index, Item, PriceItem, Stock, Tag,
                                         Type, db)
 from pystockdb.tools.data_crawler import DataCrawler
@@ -25,11 +27,12 @@ def __init__(self, arguments: dict, logger: logging.Logger):
         self.logger = logger
         self.db_args = arguments["db_args"]
         self.arguments = arguments
+        self.ticker_symbols = PyTickerSymbols()
         # has connection
         if db.provider is None:
             # prepare db
             db.bind(**self.db_args)
-            if self.arguments.get('create', False):
+            if self.db_args.get('create_db', False):
                 db.generate_mapping(check_tables=False)
                 db.drop_all_tables(with_all_data=True)
                 db.create_tables()
@@ -75,7 +78,6 @@ def __add_stock_to_index(self, index, stock_info):
             stock = Stock(name=stock_info['name'],
                           price_item=PriceItem(item=Item()))
             # add symbols
-            "".startswith
             yao = Tag.get(name=Tag.YAO)
             gog = Tag.get(name=Tag.GOG)
             usd = Tag.get(name=Tag.USD)
@@ -117,6 +119,10 @@ def download_historicals(self, symbols, start, end):
             ids = [symbol.name for symbol in chunk]
             series = crawler.get_series_stack(ids, start=start, end=end)
             for symbol in chunk:
+                self.logger.debug(
+                    'Add prices for {} from {} until {}.'.format(symbol.name,
+                                                                 start, end)
+                )
                 for value in series[symbol.name]:
                     symbol.prices.create(**value)
             commit()

src/pystockdb/tools/create.py

@@ -11,10 +11,7 @@
 import logging
 from datetime import timedelta
 
-from pony.orm import commit, db_session
-from pytickersymbols import PyTickerSymbols
-
-from pystockdb.db.schema.stocks import Symbol, Tag, Type
+from pystockdb.db.schema.stocks import Symbol, Tag
 from pystockdb.tools.base import DBBase
 
 
@@ -29,7 +26,6 @@ def __init__(self, arguments: dict, logger: logging.Logger):
         self.currency = arguments['currency']
         self.history = arguments['max_history']
         self.indices_list = arguments['indices']
-        self.ticker_symbols = PyTickerSymbols()
 
     def build(self):
         """

src/pystockdb/tools/fundamentals.py

@@ -121,12 +121,12 @@ def get_income(self, ticker_id):
 
     @returns.json
     @get('/api/securities/stock/{ticker_id}/statementsV2Detail')
-    def get_sheet(self, ticker_id, type: Query,
+    def get_sheet(self, ticker_id, sheet_type: Query("type"),
                   query_number: Query('queryNumber') = 300):
         """
         Returns financial sheet(income, balance, cash flow) data for stock
         :param ticker_id: ticker id
-        :param type: income=1, balance=2 and cashflow = 3
+        :param sheet_type: income=1, balance=2 and cashflow = 3
         :param queryNumber:
         :return:
         """

src/pystockdb/tools/update.py

@@ -68,7 +68,7 @@ def update_prices(self):
         for key in update:
             start = datetime.datetime.strptime(key, '%Y-%m-%d')
             end = datetime.datetime.now()
-            if start >= end:
+            if start.date() >= end.date():
                 continue
             self.download_historicals(update[key],
                                       start=start.strftime('%Y-%m-%d'),
@@ -89,7 +89,16 @@ def update_fundamentals(self):
         fundamentals = Fundamentals(base_url=Fundamentals.BASE_URL)
         tickers = fundamentals.get_ticker_ids(gog_syms)
         for ticker in tickers:
-            stock = [sto[0] for sto in stocks if sto[1] == ticker][0]
+            self.logger.debug(
+                'Download fundamentals for {}'.format(tickers[ticker])
+            )
+            stock = [sto[0] for sto in stocks if sto[1] == ticker]
+            if len(stock) != 1:
+                self.logger.warning(
+                    'Can not download fundamentals for {}'.format(ticker)
+                    )
+                continue
+            stock = stock[0]
             ica = fundamentals.get_income_analysis(tickers[ticker])
             ifc = fundamentals.get_income_facts(tickers[ticker])
             rec = fundamentals.get_recommendation(tickers[ticker])
@@ -99,16 +108,16 @@ def update_fundamentals(self):
             for val in [(ica, Tag.ICA), (ifc, Tag.ICF), (rec, Tag.REC),
                         (ble, Tag.BLE), (ico, Tag.ICO), (csh, Tag.CSH)]:
                 # hash stock name, tag and data
-                m = hashlib.md5()
+                m = hashlib.sha256()
                 m.update(val[1].encode('UTF-8'))
                 m.update(stock.name.encode('UTF-8'))
                 data_str = json.dumps(val[0])
                 m.update(data_str.encode('UTF-8'))
-                md5 = m.hexdigest()
+                shahash = m.hexdigest()
                 # only add data if not exist
-                if Data.get(md5=md5) is None:
+                if Data.get(hash=shahash) is None:
                     tag = Tag.get(name=val[1])
-                    obj = Data(data=val[0], md5=md5,
+                    obj = Data(data=val[0], hash=shahash,
                                data_item=DataItem(item=Item(tags=[tag])))
                     stock.data_items.add(obj.data_item)
         commit()

tests/test_crawler.py

@@ -30,16 +30,21 @@ def test_fundamentals(self):
         cash_flow = fundamentals.get_sheet('913261697', 3)
 
         search_results = fundamentals.search('Adidas')
-        assert search_results and 'tickerId' in search_results[0]
+        self.assertIsNotNone(search_results)
+        self.assertIn('tickerId', search_results[0])
         ads_id = search_results[0]['tickerId']
         income_2 = fundamentals.get_income_facts(ads_id)
         income_analyse = fundamentals.get_income_analysis(ads_id)
         recommendation = fundamentals.get_recommendation(ads_id)
         ticker = fundamentals.get_ticker_ids(TestCrawler.SYMBOLS)
-        assert len(ticker) == 5
-        assert any([symbol in ticker for symbol in TestCrawler.SYMBOLS])
-        assert any([income, balance, cash_flow, brief, income_2,
-                   income_analyse, recommendation, ticker])
+        self.assertEqual(len(ticker), 5)
+        self.assertTrue(
+          any([symbol in ticker for symbol in TestCrawler.SYMBOLS])
+        )
+        self.assertTrue(
+          any([income, balance, cash_flow, brief, income_2,
+              income_analyse, recommendation, ticker])
+        )
 
     def test_crawler(self):
         """
@@ -48,12 +53,14 @@ def test_crawler(self):
         """
         crawler = DataCrawler(True)
         price = crawler.get_last_price('ADS.F')
-        assert price
+        self.assertIsNotNone(price)
         price_stack = crawler.get_last_price_stack(TestCrawler.SYMBOLS_Y)
-        assert any([symbol in price_stack and price_stack[symbol]
-                    for symbol in TestCrawler.SYMBOLS_Y])
+        self.assertTrue(
+          any([symbol in price_stack and price_stack[symbol]
+              for symbol in TestCrawler.SYMBOLS_Y])
+        )
         ads_day = crawler.get_series('ADS.F', '1d')
-        assert ads_day
+        self.assertIsNotNone(ads_day)
 
 
 if __name__ == "__main__":

tests/test_db.py

@@ -50,10 +50,10 @@ def test_create(self):
             create_database('database_create.sqlite')
         with db_session:
             stocks = Stock.select().count()
-            assert stocks == 30
+            self.assertEqual(stocks, 30)
             prices = list(select(max(p.date) for p in Price))
-            assert len(prices) == 1
-            assert prices[0].strftime('%Y-%m-%d') == '2019-01-14'
+            self.assertEqual(len(prices), 1)
+            self.assertEqual(prices[0].strftime('%Y-%m-%d'), '2019-01-14')
 
     def test_update(self):
         """
@@ -75,17 +75,18 @@ def test_update(self):
         update.build()
         with db_session:
             prices = list(select(max(p.date) for p in Price))
-            assert len(prices) == 1
-            assert prices[0] > datetime.datetime.strptime('2019-01-14',
-                                                          '%Y-%m-%d')
+            self.assertEqual(len(prices), 1)
+            my_date = datetime.datetime.strptime('2019-01-14',
+                                                 '%Y-%m-%d')
+            self.assertGreater(prices[0], my_date)
             price_ctx = Price.select().count()
             data_ctx = Data.select().count()
         update.build()
         with db_session:
             price_ctx_now = Price.select().count()
             data_ctx_now = Data.select().count()
-            assert price_ctx_now == price_ctx
-            assert data_ctx_now == data_ctx
+            self.assertEqual(price_ctx_now, price_ctx)
+            self.assertEqual(data_ctx_now, data_ctx)
 
     def test_sync(self):
         """Tests sync tool
@@ -104,7 +105,7 @@ def test_sync(self):
         sync.build()
         with db_session:
             stocks = Stock.select().count()
-            assert stocks == 71
+            self.assertEqual(stocks, 70)
 
 
 if __name__ == "__main__":