enhancement for sqlSaver: save metrics and results, backtest only sav…

…e states to sqlSaver, all backtest result should be able to read from database instead of from memory

.pydevproject

@@ -1,9 +1,11 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<?eclipse-pydev version="1.0"?><pydev_project>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-<path>/ultra-finance/ultrafinance</path>
-<path>/ultra-finance</path>
+<path>/ultraFinance/ultrafinance</path>
+<path>/ultraFinance</path>
 </pydev_pathproperty>
 </pydev_project>

conf/backtest_period.ini

@@ -10,7 +10,7 @@ backtest.output_saver = true
 backtest.input_dam = sql
 backtest.input_db = sqlite:////data/stock.sqlite
 backtest.output_saver = sql
-backtest.output_db = sqlite:////data/output.sqlite
+backtest.output_db_prefix = sqlite:////data/
 backtest.strategy.period = 30
 
 [loggers]

conf/backtest_sma.ini

@@ -10,7 +10,7 @@ backtest.output_saver = true
 backtest.input_dam = sql
 backtest.input_db = sqlite:////data/stock.sqlite
 backtest.output_saver = sql
-backtest.output_db = sqlite:////data/output.sqlite
+backtest.output_db_prefix = sqlite:////data/
 
 [loggers]
 keys = root

conf/backtest_smaPortfolio.ini

@@ -11,7 +11,7 @@ backtest.output_saver = true
 backtest.input_dam = sql
 backtest.input_db = sqlite:////data/stock.sqlite
 backtest.output_saver = sql
-backtest.output_db = sqlite:////data/output.sqlite
+backtest.output_db_prefix = sqlite:////data/
 
 
 [loggers]

conf/backtest_zscorePortfolio.ini

@@ -11,7 +11,7 @@ backtest.output_saver = true
 backtest.input_dam = sql
 backtest.input_db = sqlite:////data/stock.sqlite
 backtest.output_saver = sql
-backtest.output_db = sqlite:////data/output.sqlite
+backtest.output_db_prefix = sqlite:////data/
 
 
 [loggers]

conf/stock.list

@@ -1 +1,2 @@
+EBAY
 GOOG

ultrafinance/backTest/constant.py

@@ -18,7 +18,7 @@
 CONF_INPUT_DB = 'backtest.input_db'
 CONF_STRATEGY_NAME = 'backtest.strategy_name'
 CONF_SAVER = 'backtest.output_saver'
-CONF_OUTPUT_DB = 'backtest.output_db'
+CONF_OUTPUT_DB_PREFIX = 'backtest.output_db_prefix'
 CONF_START_TRADE_DATE = 'backtest.start_trade_date'
 CONF_END_TRADE_DATE = 'backtest.end_trade_date'
 CONF_BUYING_RATIO = 'backtest.buying_ratio'
@@ -41,4 +41,11 @@
 STATE_SAVER_HOLDING_VALUE = "holdingValue"
 STATE_SAVER_INDEX_PRICE = "indexPrice"
 STATE_SAVER_UPDATED_ORDERS = "updatedOrders"
-STATE_SAVER_PLACED_ORDERS = "placedOrders"
+STATE_SAVER_PLACED_ORDERS = "placedOrders"
+STATE_SAVER_METRICS_START_DATE = "startDate"
+STATE_SAVER_METRICS_END_DATE = "endDate"
+STATE_SAVER_METRICS_LOWEST_VALUE = "lowestValue"
+STATE_SAVER_METRICS_HIGHEST_VALUE = "highestValue"
+STATE_SAVER_METRICS_SHARPE_RATIO = "sharpeRatio"
+STATE_SAVER_METRICS_MAX_DRAW_DOWN = "maxDrawDown"
+STATE_SAVER_METRICS_R_SQUARED = "rSquared"

ultrafinance/backTest/metric.py

@@ -85,7 +85,7 @@ def formatResult(self):
              self.result[BasicMetric.MAX_DRAW_DOWN][1], self.result[BaseMetric.MAX_DRAW_DOWN][0],
              self.result[BasicMetric.SRATIO], self.result[BasicMetric.R_SQUARED])
 
-class MetricCalculator(object):
+class MetricManager(object):
     ''' TODO: make it more generic for more metrics '''
     def __init__(self):
         ''' constructor '''
@@ -96,6 +96,7 @@ def calculate(self, symbols, timePositions, iTimePositionDict):
         metric = BasicMetric()
         metric.calculate(timePositions, iTimePositionDict)
         self.__calculated['_'.join(symbols)] = metric.result
+        return metric.result
 
     def formatMetrics(self):
         ''' output all calculated metrics '''

ultrafinance/backTest/stateSaver/sqlSaver.py

@@ -10,7 +10,7 @@
 from sqlalchemy import Table, Column, Integer, String, create_engine, MetaData, and_, select
 from sqlalchemy.ext.declarative import declarative_base
 
-from ultrafinance.backTest.constant import STATE_SAVER_ACCOUNT, STATE_SAVER_UPDATED_ORDERS, STATE_SAVER_PLACED_ORDERS, STATE_SAVER_HOLDING_VALUE, STATE_SAVER_INDEX_PRICE
+from ultrafinance.backTest.constant import *
 
 import sys
 import json
@@ -42,14 +42,18 @@ def __str__(self):
 
 class SqlSaver(StateSaver):
     ''' sql saver '''
-    COLUMNS = [Column('time', Integer, primary_key = True),
-               Column(STATE_SAVER_ACCOUNT, String(40)),
-               Column(STATE_SAVER_INDEX_PRICE, String(40)),
-               Column(STATE_SAVER_UPDATED_ORDERS, String(200)),
-               Column(STATE_SAVER_HOLDING_VALUE, String(40)),
-               Column(STATE_SAVER_PLACED_ORDERS, String(200))]
+    RESULT_COLUMNS = [Column('time', Integer, primary_key = True),
+                      Column(STATE_SAVER_ACCOUNT, String(40)),
+                      Column(STATE_SAVER_INDEX_PRICE, String(40)),
+                      Column(STATE_SAVER_UPDATED_ORDERS, String(200)),
+                      Column(STATE_SAVER_HOLDING_VALUE, String(40)),
+                      Column(STATE_SAVER_PLACED_ORDERS, String(200))]
 
-    EXISTED_TABLE_DICT = {}
+    METRICS_COLUMNS = [Column('metric', String(40), primary_key = True),
+                       Column('value', String(40))]
+
+    RESULT_TABLE = None
+    METRICS_TABLE = None
 
     def __init__(self):
         ''' constructor '''
@@ -61,38 +65,44 @@ def __init__(self):
         #}
         super(SqlSaver, self).__init__()
         self.__writeCache = {}
-        self.table = None
+        self.__metrics = []
         self.db = None
-        self.metadata = MetaData()
+        self.metadata = None
         self.engine = None
-        self.firstTime = True
 
-    def setup(self, setting, tableName):
+    def setup(self, setting):
         ''' setup '''
         if 'db' in setting:
             self.db = setting['db']
             self.engine = create_engine(setting['db'], echo = False)
-            self.tableName = tableName
-            self.__constructTable()
-            self.table = SqlSaver.EXISTED_TABLE_DICT.get(tableName)
+            self.metadata = MetaData(self.engine)
+            self.__constructTableIfNotExist()
 
-    def __constructTable(self):
+    def __constructTableIfNotExist(self):
         ''' construct table '''
-        if not self.firstTime or self.table is not None or not self.tableName \
-        or self.tableName in SqlSaver.EXISTED_TABLE_DICT:
-            return
+        if SqlSaver.RESULT_TABLE is None:
+            SqlSaver.RESULT_TABLE = Table("result",
+                                          self.metadata,
+                                          *SqlSaver.RESULT_COLUMNS)
+
+        if SqlSaver.METRICS_TABLE is None:
+            SqlSaver.METRICS_TABLE = Table("metrics",
+                                           self.metadata,
+                                           *SqlSaver.METRICS_COLUMNS)
 
-        SqlSaver.EXISTED_TABLE_DICT[self.tableName] = Table(self.tableName,
-                                                            self.metadata,
-                                                            *SqlSaver.COLUMNS)
+    def __resetResultTable(self):
+        ''' reset result table '''
+        SqlSaver.RESULT_TABLE.drop(self.engine, checkfirst = True)
 
+        LOG.info("create db %s with cols %s" % (self.db, SqlSaver.RESULT_COLUMNS))
+        SqlSaver.RESULT_TABLE.create(self.engine, checkfirst = True)
 
-    def resetTable(self):
-        ''' create cols '''
-        self.table.drop(self.engine, checkfirst = True)
+    def __resetMetricsTable(self):
+        ''' reset result table '''
+        SqlSaver.METRICS_TABLE.drop(self.engine, checkfirst = True)
 
-        LOG.info("create db %s with table %s with cols %s" % (self.db, self.tableName, SqlSaver.COLUMNS))
-        self.table.create(self.engine, checkfirst = True)
+        LOG.info("create db %s with cols %s" % (self.db, SqlSaver.METRICS_COLUMNS))
+        SqlSaver.METRICS_TABLE.create(self.engine, checkfirst = True)
 
     def getStates(self, start, end):
         ''' read value for a col  '''
@@ -103,10 +113,20 @@ def getStates(self, start, end):
             end = sys.maxint
 
         conn = self.engine.connect()
-        rows = conn.execute(select([self.table]).where(and_(self.table.c.time >= int(start),
-                                                            self.table.c.time < int(end))))
+        rows = conn.execute(select([self.resultTable]).where(and_(SqlSaver.RESULT_TABLE.c.time >= int(start),
+                                                                  SqlSaver.RESULT_TABLE.c.time < int(end))))
 
-        return [self.__tupleToResult(row) for row in rows]
+        #return [self.__tupleToResult(row) for row in rows]
+        ret = []
+        for row in rows:
+            ret.append({"time": row['time'],
+                        STATE_SAVER_ACCOUNT: row[STATE_SAVER_ACCOUNT],
+                        STATE_SAVER_HOLDING_VALUE: row[STATE_SAVER_HOLDING_VALUE],
+                        STATE_SAVER_INDEX_PRICE: row[STATE_SAVER_INDEX_PRICE],
+                        STATE_SAVER_UPDATED_ORDERS: row[STATE_SAVER_UPDATED_ORDERS],
+                        STATE_SAVER_PLACED_ORDERS: row[STATE_SAVER_UPDATED_ORDERS]})
+
+        return ret
 
     def __tupleToResult(self, row):
         ''' convert tuple queried from sql to BackTestResult'''
@@ -121,6 +141,19 @@ def __tupleToResult(self, row):
             LOG.error("Unknown exception doing __tupleToResult in sqlSaver " + str(ex) + " --row-- " + str(row))
             return BackTestResult('-1', '-1', '-1', '-1', '[]', '[]')
 
+    def getMetrics(self, start, end):
+        ''' read value for a col  '''
+        d = {}
+        if self.engine is None:
+            return d
+
+        conn = self.engine.connect()
+        rows = conn.execute(select([SqlSaver.METRICS_TABLE]))
+
+        for row in rows:
+            d[row['metric']] = row['value']
+
+        return d
 
     def __sqlToResult(self, row):
         ''' convert row result '''
@@ -137,13 +170,35 @@ def write(self, timestamp, col, value):
             self.__writeCache[timestamp] = {}
         self.__writeCache[timestamp][col] = value
 
+    def writeMetrics(self, startDate, endDate, lowestValue, highestValue, sharpeRatio, maxDrawDown, rSquared, endValue, endHoldings):
+        ''' write metrics '''
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_START_DATE, 'value': str(startDate)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_END_DATE, 'value': str(endDate)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_LOWEST_VALUE, 'value': str(lowestValue)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_HIGHEST_VALUE, 'value': str(highestValue)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_SHARPE_RATIO, 'value': str(sharpeRatio)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_MAX_DRAW_DOWN, 'value': str(maxDrawDown)})
+        self.__metrics.append({'metric': STATE_SAVER_METRICS_R_SQUARED, 'value': str(rSquared)})
+        self.__metrics.append({'metric': "endValue", 'value': str(endValue)})
+        self.__metrics.append({'metric': "endHoldings", 'value': json.dumps(self.__convertHoldingsToList(endHoldings))})
+
+
+    def __convertHoldingsToList(self, holding):
+        ''' convert holding to dict'''
+        ret = []
+        for symbol, (share, price) in holding.items():
+            if share <= 0:
+                continue
+
+            ret.append({"symbol": symbol,
+                        "share": int(share),
+                        "price": "%.2f" % price})
+
+        return ret
+
     def commit(self):
         ''' complete write operation '''
-        if not self.tableName:
-            raise UfException(Errors.TABLENAME_NOT_SET,
-                              "Table name not set")
-
-        self.resetTable()
+        self.__resetResultTable()
 
         # write values
         updates = []
@@ -168,8 +223,13 @@ def commit(self):
             updates.append(update)
 
         conn = self.engine.connect()
-        conn.execute(self.table.insert(), updates)
-        LOG.info("committed table %s at %s" % (self.table, self.db))
+        conn.execute(SqlSaver.RESULT_TABLE.insert(), updates)
+
+        if self.__metrics:
+            self.__resetMetricsTable()
+            conn.execute(SqlSaver.METRICS_TABLE.insert(), self.__metrics)
+            self.__metrics = []
+        LOG.info("committed table result at %s" % self.db)
 
 def listTableNames(db):
     ''' list names of tables '''
@@ -183,12 +243,19 @@ def listTableNames(db):
         return []
 
 
+
+
 if __name__ == '__main__':
-    s = SqlSaver()
-    s.setup({'db': 'sqlite:////data/test_output.sqlite'}, 'unittest_outputSaver')
-    #h.resetCols(['accountValue', '1'])
-    for i in range(5):
-        s.write(123456, STATE_SAVER_HOLDING_VALUE, 10000)
-        s.commit()
-
-    print listTableNames("sqlite:////data/output.sqlite")
+    for i in range(2):
+        s = SqlSaver()
+        s.setup({'db': 'sqlite:////data/test_output' + str(i) + '.sqlite'})
+        #h.resetCols(['accountValue', '1'])
+        for i in range(5):
+            s.write(123456, STATE_SAVER_HOLDING_VALUE, 10000)
+            s.write(123456, STATE_SAVER_ACCOUNT, 12312312)
+            s.write(123456, STATE_SAVER_INDEX_PRICE, 123123)
+
+            s.writeMetrics(2010, 2013, 1, 100, 10, 0.1, 0.1, 123123, {})
+            s.commit()
+
+        print listTableNames("sqlite:////data/test_output" + str(i) + ".sqlite")

ultrafinance/backTest/stateSaver/stateSaverFactory.py

@@ -12,17 +12,14 @@
 class StateSaverFactory(Singleton):
     ''' factory for output saver '''
     @staticmethod
-    def createStateSaver(name, setting, tableName = None):
+    def createStateSaver(name, setting, tableName):
         ''' create state saver '''
-        if 'habse' == name:
-            from ultrafinance.backTest.stateSaver.hbaseSaver import HbaseSaver
-            saver = HbaseSaver()
-        elif 'sql' == name:
+        if 'sql' == name:
             from ultrafinance.backTest.stateSaver.sqlSaver import SqlSaver
             saver = SqlSaver()
         else:
             raise UfException(Errors.INVALID_SAVER_NAME,
                               "Saver name is invalid %s" % name)
 
-        saver.setup(setting, 'output' if tableName is None else tableName)
+        saver.setup(setting)
         return saver

ultrafinance/backTest/tickSubscriber/strategies/smaPortfolioStrategy.py

@@ -78,7 +78,7 @@ def __init__(self, symbol, strategy, buyingRatio):
 
         self.__smaShort = Sma(10)
         self.__smaMid = Sma(60)
-        self.__smaLong = Sma(300)
+        self.__smaLong = Sma(200)
         self.__smaVolumeShort = Sma(10)
         self.__smaVolumeMid = Sma(60)
         self.__movingLowShort = MovingLow(10)

ultrafinance/backTest/tradingEngine.py

@@ -111,11 +111,6 @@ def _complete(self):
             for sub in subDict.iterkeys():
                 sub.complete()
 
-        #write to saver
-        if self.saver:
-            LOG.debug("Writing state to saver")
-            self.saver.commit()
-
     def consumeTicks(self, ticks, sub, event):
         ''' publish ticks to sub '''
         thread = Thread(target = getattr(sub, event), args = (ticks,))

ultrafinance/dam/sqlDAM.py

@@ -11,7 +11,6 @@
 from sqlalchemy import Column, Integer, String, Float, Sequence, create_engine, and_
 from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
-import time
 
 Base = declarative_base()