Merge pull request #164 from shlomikushchi/postgres_backend

Adding a postgres backend and alpha_vantage bundle.

.github/workflows/ci.yaml

@@ -68,4 +68,4 @@ jobs:
         python -m pip install --no-binary=bcolz -e .[all] -r etc/requirements_blaze.in
     - name: Run tests
       run: |
-        nosetests tests
+        nosetests tests

config/__init__.py

@@ -0,0 +1 @@
+from config import bundle

config/bundle.py

@@ -0,0 +1,126 @@
+import os
+
+import yaml
+
+CONFIG_PATH = os.environ.get("ZIPLINE_TRADER_CONFIG")
+if CONFIG_PATH:
+    with open(CONFIG_PATH, mode='r') as f:
+        ZIPLINE_CONFIG = yaml.safe_load(f)
+
+
+class AlpacaConfig:
+    if CONFIG_PATH:
+        al = ZIPLINE_CONFIG["alpaca"]
+
+    @property
+    def key(self):
+        if CONFIG_PATH:
+            return self.al["key_id"]
+        else:
+            return os.environ.get('APCA_API_KEY_ID')
+
+    @property
+    def secret(self):
+        if CONFIG_PATH:
+            return self.al["secret"]
+        else:
+            return os.environ.get('APCA_API_SECRET_KEY')
+
+    @property
+    def base_url(self):
+        if CONFIG_PATH:
+            return self.al["base_url"]
+        else:
+            return os.environ.get('APCA_API_BASE_URL')
+
+    @property
+    def universe(self):
+        if CONFIG_PATH:
+            return self.al["universe"]
+        else:
+            return os.environ.get('ZT_UNIVERSE')
+
+    @property
+    def custom_asset_list(self):
+        if CONFIG_PATH:
+            return self.al["custom_asset_list"]
+        else:
+            return os.environ.get('ZT_CUSTOM_ASSET_LIST')
+
+
+
+class AlphaVantage:
+    if CONFIG_PATH:
+        av = ZIPLINE_CONFIG["alpha-vantage"]
+
+    @property
+    def sample_frequency(self):
+        """
+        how long to wait between samples. default for free accounts - 1 min.
+        so we could do 5 sample per minute.
+        you could define it in the config file or override it with env variable
+        :return:
+        """
+        val = 60
+        if os.environ.get('AV_FREQ_SEC'):
+            val = int(os.environ.get('AV_FREQ_SEC'))
+        elif CONFIG_PATH and self.av.get('AV_FREQ_SEC'):
+            val = int(self.av.get('AV_FREQ_SEC'))
+        return val
+
+    @property
+    def max_calls_per_freq(self):
+        """
+        max api calls you could do per frequency period.
+        free account can do 5 calls per minute
+        you could define it in the config file or override it with env variable
+        :return:
+        """
+        val = 5
+        if os.environ.get('AV_CALLS_PER_FREQ'):
+            val = int(os.environ.get('AV_CALLS_PER_FREQ'))
+        elif CONFIG_PATH and self.av.get('AV_CALLS_PER_FREQ'):
+            val = int(self.av.get('AV_CALLS_PER_FREQ'))
+        return val
+
+    @property
+    def breathing_space(self):
+        """
+        to make sure we don't pass the limit we take some breathing room for sampling error.
+        you could define it in the config file or override it with env variable
+        :return:
+        """
+        val = 1
+        if os.environ.get('AV_TOLERANCE_SEC'):
+            val = int(os.environ.get('AV_TOLERANCE_SEC'))
+        elif CONFIG_PATH and self.av.get('AV_TOLERANCE_SEC'):
+            val = int(self.av.get('AV_TOLERANCE_SEC'))
+        return val
+
+    @property
+    def api_key(self):
+        """
+        api key for alpha vantage
+        you could define it in the config file or override it with env variable
+        :return:
+        """
+        val = ''
+        if os.environ.get('ALPHAVANTAGE_API_KEY'):
+            val = os.environ.get('ALPHAVANTAGE_API_KEY')
+        elif CONFIG_PATH and self.av.get('ALPHAVANTAGE_API_KEY'):
+            val = self.av.get('ALPHAVANTAGE_API_KEY')
+        return val
+
+
+def get_binance_config():
+    if CONFIG_PATH:
+        return ZIPLINE_CONFIG["binance"]
+
+
+if __name__ == '__main__':
+    print(ZIPLINE_CONFIG)
+    print(AlpacaConfig().key)
+    av_conf = AlphaVantage()
+    print(av_conf.sample_frequency)
+    print(av_conf.max_calls_per_freq)
+    print(get_binance_config())

config/data_backend.py

@@ -0,0 +1,113 @@
+import os
+
+import yaml
+
+CONFIG_PATH = os.environ.get("ZIPLINE_TRADER_CONFIG")
+if CONFIG_PATH:
+    with open(CONFIG_PATH, mode='r') as f:
+        ZIPLINE_CONFIG = yaml.safe_load(f)
+
+
+def db_backend_configured():
+    if CONFIG_PATH:
+        return ZIPLINE_CONFIG["backend"].get("type", False)
+    else:
+        return os.environ.get('ZIPLINE_DATA_BACKEND')
+
+
+class PostgresDB:
+    if CONFIG_PATH:
+        pg = ZIPLINE_CONFIG["backend"]["postgres"]
+
+    @property
+    def host(self):
+        """
+        you could define it in the zipline-trader config file or
+        override it with this env variable: ZIPLINE_DATA_BACKEND_HOST
+        :return:
+        """
+        val = None
+        if os.environ.get('ZIPLINE_DATA_BACKEND_HOST'):
+            val = os.environ.get('ZIPLINE_DATA_BACKEND_HOST')
+        elif CONFIG_PATH and self.pg.get('host'):
+            val = self.pg.get('host')
+        if not val:
+            raise Exception("Postgres host not defined by user")
+        return val
+
+    @property
+    def port(self):
+        """
+        you could define it in the zipline-trader config file or
+        override it with this env variable: ZIPLINE_DATA_BACKEND_PORT
+        :return:
+        """
+        val = None
+        if os.environ.get('ZIPLINE_DATA_BACKEND_PORT'):
+            val = os.environ.get('ZIPLINE_DATA_BACKEND_PORT')
+        elif CONFIG_PATH and self.pg.get('port'):
+            val = self.pg.get('port')
+        if not val:
+            raise Exception("Postgres port not defined by user")
+        return int(val)
+
+    @property
+    def user(self):
+        """
+        you could define it in the zipline-trader config file or
+        override it with this env variable: ZIPLINE_DATA_BACKEND_USER
+        :return:
+        """
+        val = None
+        if os.environ.get('ZIPLINE_DATA_BACKEND_USER'):
+            val = os.environ.get('ZIPLINE_DATA_BACKEND_USER')
+        elif CONFIG_PATH and self.pg.get('user'):
+            val = self.pg.get('user')
+        if not val:
+            raise Exception("Postgres user not defined by user")
+        return val
+
+    @property
+    def password(self):
+        """
+        you could define it in the zipline-trader config file or
+        override it with this env variable: ZIPLINE_DATA_BACKEND_PASSWORD
+        :return:
+        """
+        val = None
+        if os.environ.get('ZIPLINE_DATA_BACKEND_PASSWORD'):
+            val = os.environ.get('ZIPLINE_DATA_BACKEND_PASSWORD')
+        elif CONFIG_PATH and self.pg.get('password'):
+            val = self.pg.get('password')
+        if not val:
+            raise Exception("Postgres password not defined by user")
+        return val
+
+
+if __name__ == '__main__':
+    print(ZIPLINE_CONFIG)
+    db = PostgresDB()
+    print(db.host)
+    os.environ["ZIPLINE_DATA_BACKEND_HOST"] = "localhost"
+    print(db.host)
+    del db.pg["host"]
+    del os.environ["ZIPLINE_DATA_BACKEND_HOST"]
+    try:
+        print(db.host)
+    except Exception as e:
+        print(e)
+
+    print(db.port)
+    os.environ["ZIPLINE_DATA_BACKEND_PORT"] = "5433"
+    assert 5433 == db.port
+
+    print(db.user)
+    os.environ["ZIPLINE_DATA_BACKEND_USER"] = "userrr"
+    assert "userrr" == db.user
+
+    print(db.password)
+    os.environ["ZIPLINE_DATA_BACKEND_PASSWORD"] = "passdd"
+    assert "passdd" == db.password
+
+
+

etc/requirements.in

@@ -74,4 +74,9 @@ bcolz==1.2.1
 # exact version
 h5py==2.10.0
 html5lib==1.1
-beautifulsoup4==4.9.3
+beautifulsoup4==4.9.3
+# psql-backend
+psycopg2==2.8.6
+# alpha-vantage-bundle
+alpha_vantage==2.3.1
+ratelimit==2.2.1

zipline/__main__.py

@@ -1,5 +1,7 @@
 import errno
 import os
+import pkgutil
+from importlib import import_module
 
 import click
 import logbook

zipline/assets/asset_db_migrations.py

@@ -7,7 +7,7 @@
 from zipline.utils.compat import wraps
 from zipline.errors import AssetDBImpossibleDowngrade
 from zipline.utils.preprocess import preprocess
-from zipline.utils.sqlite_utils import coerce_string_to_eng
+from zipline.utils.db_utils import coerce_string_to_eng
 
 
 def alter_columns(op, name, *columns, **kwargs):

zipline/assets/asset_db_schema.py

@@ -1,6 +1,5 @@
 import sqlalchemy as sa
 
-
 # Define a version number for the database generated by these writers
 # Increment this version number any time a change is made to the schema of the
 # assets database
@@ -42,16 +41,16 @@
     metadata,
     sa.Column(
         'sid',
-        sa.Integer,
+        sa.BigInteger,
         unique=True,
         nullable=False,
         primary_key=True,
     ),
     sa.Column('asset_name', sa.Text),
-    sa.Column('start_date', sa.Integer, default=0, nullable=False),
-    sa.Column('end_date', sa.Integer, nullable=False),
-    sa.Column('first_traded', sa.Integer),
-    sa.Column('auto_close_date', sa.Integer),
+    sa.Column('start_date', sa.BigInteger, default=0, nullable=False),
+    sa.Column('end_date', sa.BigInteger, nullable=False),
+    sa.Column('first_traded', sa.BigInteger),
+    sa.Column('auto_close_date', sa.BigInteger),
     sa.Column('exchange', sa.Text, sa.ForeignKey(exchanges.c.exchange)),
 )
 
@@ -88,12 +87,12 @@
     ),
     sa.Column(
         'start_date',
-        sa.Integer,
+        sa.BigInteger,
         nullable=False,
     ),
     sa.Column(
         'end_date',
-        sa.Integer,
+        sa.BigInteger,
         nullable=False,
     ),
 )