Optimizer

analysis/notebooks/hyperparameters_analysis.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Take Action: Select portfolio for analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optimizer_run_id = \"c76c97cb-d8cd-4498-8112-d75b2b1f1443\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import empyrical as ep\n",
+    "import math\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import quantstats as qs\n",
+    "from IPython.display import HTML, display, Markdown\n",
+    "from liualgotrader.analytics import analysis\n",
+    "import matplotlib.ticker as mtick\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Load data details"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hyperparameters = analysis.calc_hyperparameters_analysis(optimizer_run_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "analysis = pd.DataFrame(columns=['profile_id'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "analysis['profile_id'] = hyperparameters.index.levels[0]\n",
+    "analysis['sharpe'] = analysis['profile_id'].apply(\n",
+    "    lambda x: qs.stats.sharpe(\n",
+    "        returns=hyperparameters[\n",
+    "            hyperparameters.index.get_level_values(\"portfolio_id\") == x\n",
+    "        ]\n",
+    "    ).totals\n",
+    ")\n",
+    "analysis['volatility'] = analysis['profile_id'].apply(\n",
+    "    lambda x: qs.stats.volatility(\n",
+    "        returns=hyperparameters[\n",
+    "            hyperparameters.index.get_level_values(\"portfolio_id\") == x\n",
+    "        ]\n",
+    "    ).totals\n",
+    ")\n",
+    "analysis['total'] = analysis['profile_id'].apply(\n",
+    "    lambda x: hyperparameters[\n",
+    "            hyperparameters.index.get_level_values(\"portfolio_id\") == x\n",
+    "        ]\n",
+    "    .totals[-1]\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "analysis"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

analysis/notebooks/portfolio_analysis.ipynb

@@ -13,7 +13,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "portfolio_id = \"46f1e6f3-cf3e-4902-9225-0b0e07f4a547\"\n",
+    "portfolio_id = \"765c2f7d-658f-4e29-a276-11fc904cdff9\"\n",
     "report_title = \"Trade JNUG/JDST (based on GDXJ SMA)volatility,  Jun-20 to Jun-21 \""
    ]
   },

database/schema.sql

@@ -328,3 +328,17 @@ ALTER TABLE trending_tickers ADD COLUMN scanner_name text NOT NULL DEFAULT 'mome
 CREATE INDEX ON trending_tickers (scanner_name);
 CREATE INDEX ON trending_tickers (create_tstamp);
 ALTER TABLE  trending_tickers ALTER COLUMN create_tstamp TYPE timestamp with time zone;
+
+
+
+CREATE TABLE IF NOT EXISTS optimizer_run (
+    optimizer_run_id serial PRIMARY KEY,
+    optimizer_session_id text NOT NULL,
+    batch_id text NOT NULL,
+    tstamp timestamp with time zone DEFAULT current_timestamp
+);
+
+CREATE INDEX ON optimizer_run(optimizer_session_id);
+
+
+ALTER TABLE optimizer_run ADD COLUMN parameters text NOT NULL DEFAULT('');

docs/source/Optimizer.rst

@@ -0,0 +1,62 @@
+Optimizer
+=========
+
+The `optimizer` application is a tool for back-testing strategies, in parallel, while using different parameters. 
+The optimizer helps selecting the best performing hyper-parameters for any strategy.
+
+How-To
+------
+
+To use the `optimizer` application, a new section should be added to the `tradeplan.toml` file:
+
+.. literalinclude:: ../../examples/optimizer/tradeplan.toml
+  :language: python
+  :linenos:
+
+* Note the [optimizer] section added to the tradeplan.toml file,
+* start_date & end_date specify the time-window for backtesting, 
+* The `optimizer` always runs in daily time-scale,
+
+
+Params and Hypers
+-----------------
+
+The `optimizer` application supports two different 
+types of configuration-space constructs: Parmeter and Hyper-Parameters.
+
+Parameters are meta-instructions for generating configurations for strategies,
+that are calculated per back-testing step. In contrast, Hyper-Parameters is a 
+configuration space spanning a product of all possible values for each Hyper-Parameter. 
+
+Usage
+-----
+
+To run the `optimizer` application, simply type: 
+
+.. code-block:: bash
+
+    optimizer
+
+The application will read the `tradeplan.toml` file and spawn process for backtesting the different configurations. 
+
+Model
+-----
+Each `optimizer` execution generates a UUID representing a unique exeuction identification. 
+Each back-test will generate it's own `batch-id`. 
+The relationship between the optimizer execution id, batch-ids and the hyper parameters configurations
+is stored in the **OPTIMIZER_RUN** database table.
+
+Analysis
+--------
+A dedicated Jupter Notebook (see the `analysis` section) allows loading and comparing
+performance of the different configurations.
+
+Parallel execution
+------------------
+Similarly to the `trader` application, the `optimizer` also supports multi-processing execution to optimize run times. By default the `optimizer` executes 4 process, however that may be changed by speciying the `--concurrency` parameter.
+
+
+
+
+
+

docs/source/What's New.rst

@@ -3,6 +3,17 @@ What's New
 +------------------+----------------------------------------------+
 | Release          | Notes                                        |
 +------------------+----------------------------------------------+
+| 0.3.10           | **Major Release**                            |
+|                  |                                              |
+|                  | 1. Release `optimizer` application for       |
+|                  |    hyper-parameters optimization,            |
+|                  | 2. Notebook for optimization session         |
+|                  |    review and selection of parameters,       |
+|                  | 3. Deployment & Installation fixes,          |
+|                  |    transition to travis-ci.com, cleanup      |
+|                  |    to pytests,                               |
+|                  | 4. Bug-fixes and code improvements.          |
++------------------+----------------------------------------------+
 | 0.2.00           | **Major Release**                            |
 |                  |                                              |
 |                  | 1. Add support for Account managment,        |

docs/source/index.rst

@@ -18,6 +18,7 @@ LiuAlgoTrader is made of the following components:
 - **trader**: the main application for real-time trading,
 - **backtester**: re-running past trading sessions, back-testing strategies,
 - **market_miner**: application for collecting market data and run market off-hours calculations & strategies,
+- **optimizer**: application for optimizing strategies' hyper-parameters. 
 
 AND
 
@@ -42,6 +43,7 @@ sessions w/ LiuAlgoTrader.
    Trading
    Backtesting 
    Off-market
+   Optimizer
    Analysis
    Examples
    How to Contribute

examples/optimizer/tradeplan.toml

@@ -0,0 +1,25 @@
+# This is a TOML configuration file.
+bypass_market_schedule = false
+portfolio_value = 25000
+
+[strategies]
+
+[strategies.TrendFollow]
+debug = true
+filename = "/Users/amichayoren/dev/trades/strats/trendfollow.py"
+index = 'SP500'
+portfolio_id = "2b94a5c6-4ccf-480c-91cd-71cdeacf158c"
+rank_days = 90
+rebalance_rate = "daily"
+reinvest = true
+stock_count = 10
+
+[optimizer]
+end_date = '2021-06-19'
+start_date = '2021-01-01'
+
+[optimizer.parameters.strategies.TrendFollow]
+portfolio_id = {param_type = 'portfolio', size = 10000, credit = 1000}
+
+[optimizer.hyperparameters.strategies.TrendFollow]
+stock_count = {param_type = 'int', min = 5, max = 15}

liualgotrader/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.7"
+__version__ = "0.3.10"

liualgotrader/analytics/analysis.py

@@ -12,6 +12,7 @@
 from liualgotrader.common.database import fetch_as_dataframe
 from liualgotrader.common.tlog import tlog
 from liualgotrader.models.accounts import Accounts
+from liualgotrader.models.optimizer import OptimizerRun
 from liualgotrader.models.portfolio import Portfolio
 from liualgotrader.trading.trader_factory import trader_factory
 
@@ -471,7 +472,7 @@ def get_cash(account_id: int, initial_cash: float) -> pd.DataFrame:
     df = loop.run_until_complete(Accounts.get_transactions(account_id))
     df = df.groupby(df.index.date).sum()
     df.iloc[0].amount += initial_cash
-    r = pd.date_range(start=df.index.min(), end=df.index.max())
+    r = pd.date_range(start=df.index.min(), end=df.index.max(), name="date")
     df.reindex(r).fillna(method="ffill")
     df["cash"] = df.amount.cumsum()
     df.index.rename("date", inplace=True)
@@ -497,7 +498,9 @@ def calc_portfolio_returns(portfolio_id: str) -> pd.DataFrame:
 
     cash_df = get_cash(account_id, initial_account_size)
     symbols = trades.symbol.unique().tolist()
-    for i in tqdm(range(len(symbols))):
+    for i in tqdm(
+        range(len(symbols)), desc=f"loading portfolio {portfolio_id}"
+    ):
         symbol = symbols[i]
         symbol_trades = trades[trades.symbol == symbol].sort_values(
             by="client_time"
@@ -507,3 +510,22 @@ def calc_portfolio_returns(portfolio_id: str) -> pd.DataFrame:
     td = td.fillna(method="ffill")
     td["totals"] = td["equity"] + td["cash"]
     return pd.DataFrame(td, columns=["equity", "cash", "totals"])
+
+
+def calc_hyperparameters_analysis(optimizer_run_id: str) -> pd.DataFrame:
+    loop = asyncio.get_event_loop()
+
+    portfolio_ids = loop.run_until_complete(
+        OptimizerRun.get_portfolio_ids(optimizer_run_id)
+    )
+
+    df = None
+    if len(portfolio_ids):
+        for i in tqdm(range(len(portfolio_ids)), desc="Loading Portfolios"):
+            _df = calc_portfolio_returns(portfolio_ids[i])
+            _df["portfolio_id"] = portfolio_ids[i]
+            _df.reset_index(inplace=True)
+            _df = _df.set_index(["portfolio_id", "date"])
+            df = pd.concat([df, _df], axis=0) if df is not None else _df
+
+    return df

liualgotrader/common/data_loader.py

@@ -319,7 +319,7 @@ def fetch_data_timestamp(self, timestamp: pd.Timestamp) -> None:
 
         if type(timestamp) == pd.Timestamp:
             _start = timestamp.to_pydatetime() - timedelta(
-                days=6 if self.scale == TimeScale.minute else 500
+                days=6 if self.scale == TimeScale.minute else 100
             )
             _end = timestamp.to_pydatetime() + timedelta(days=1)
         elif type(timestamp) == int:
@@ -337,11 +337,11 @@ def fetch_data_timestamp(self, timestamp: pd.Timestamp) -> None:
                     second=0, microsecond=0
                 ) + timedelta(days=1 + timestamp)
             _start = _end - timedelta(
-                days=6 if self.scale == TimeScale.minute else 500
+                days=6 if self.scale == TimeScale.minute else 100
             )
         else:
             _start = timestamp - timedelta(
-                days=6 if self.scale == TimeScale.minute else 500
+                days=6 if self.scale == TimeScale.minute else 100
             )
             _end = timestamp + timedelta(days=1)
 
@@ -381,13 +381,13 @@ def fetch_data_range(self, start: datetime, end: datetime) -> None:
                 _start = (
                     end
                     - timedelta(
-                        days=7 if self.scale == TimeScale.minute else 500
+                        days=7 if self.scale == TimeScale.minute else 100
                     )
                 ).date()
                 _end = end.date()
             else:
                 _start = end - timedelta(
-                    days=7 if self.scale == TimeScale.minute else 500
+                    days=7 if self.scale == TimeScale.minute else 100
                 )
                 _end = end
 
@@ -400,7 +400,7 @@ def fetch_data_range(self, start: datetime, end: datetime) -> None:
 
             new_df = pd.concat([_df, new_df], sort=True).drop_duplicates()
 
-            end -= timedelta(days=7 if self.scale == TimeScale.minute else 500)
+            end -= timedelta(days=7 if self.scale == TimeScale.minute else 100)
 
         # new_df = new_df[~new_df.index.duplicated(keep="first")]
         self.symbol_data = pd.concat(