Additive index

README.md

@@ -54,7 +54,7 @@ To appear
 # Requirements
 
 Major requirements
-* Required: Python 3.6
+* Required: Python 3.7
 * Required: pandas, numpy etc.
 * Recommended: blpapi - Bloomberg Python Open API
 * Recommended: chartpy - for funky interactive plots ([https://github.com/cuemacro/chartpy](https://github.com/cuemacro/chartpy)) and
@@ -125,6 +125,10 @@ individual data providers)
 
 # Coding log
 
+* 11 Nov 2020
+    * Added cumulative additive index returns 
+    * Removed log as field variable in DataVendorBBG
+    * Added 10am NYC cut for FX vol surface download
 * 02 Oct 2020
     * Fix vol ticker mapping for 4M points
     * Fix Bloomberg downloader for events

findatapy/conf/fx_vol_tickers_maker.csv

@@ -30,4 +30,3 @@ fx-implied-vol.bloomberg.daily.close,EURUSD,NYC.CMPN Curncy,,V,ON
 ,XAUUSD,,,,
 ,XAGUSD,,,,
 ,XPTUSD,,,,
-,EURCZK,,,,

findatapy/market/__init__.py

@@ -5,6 +5,6 @@
 # don't include DataVendorBBG, in case users haven't installed blpapi
 # from findatapy.market.datavendorbbg import DataVendorBBG
 from findatapy.market.ioengine import IOEngine, SpeedCache
-from findatapy.market.market import Market, FXVolFactory, FXCrossFactory, FXConv
+from findatapy.market.market import Market, FXVolFactory, FXCrossFactory, FXConv, RatesFactory
 from findatapy.market.marketdatagenerator import MarketDataGenerator
 from findatapy.market.marketdatarequest import MarketDataRequest

findatapy/market/market.py

@@ -76,7 +76,7 @@ def fetch_market(self, md_request=None):
         data_frame = None
 
         # If internet_load has been specified don't bother going to cache (might end up calling lower level cache though
-        # through MarketDataGenerator
+        # through MarketDataGenerator)
         if 'cache_algo' in md_request.cache_algo:
             data_frame = self.speed_cache.get_dataframe(key)
 
@@ -122,15 +122,15 @@ def fetch_market(self, md_request=None):
                     for t in md_request.tickers:
                         if len(t) == 6:
                             df.append(
-                                fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor,
+                                fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor_default,
                                                         cut=md_request.cut, data_source=md_request.data_source,
-                                                        part=fxvf.part,
+                                                        part=fxvf.part_default,
                                                         cache_algo=md_request.cache_algo))
 
                     if df != []:
                         data_frame = Calculations().pandas_outer_join(df)
 
-            # For FX vol market return all the market data necessarily for pricing options
+            # For FX vol market return all the market data necessary for pricing options
             # which includes FX spot, volatility surface, forward points, deposit rates
             if (md_request.category == 'fx-vol-market'):
                 if md_request.tickers is not None:
@@ -143,6 +143,7 @@ def fetch_market(self, md_request=None):
                     # For each FX cross fetch the spot, vol and forward points
                     for t in md_request.tickers:
                         if len(t) == 6:
+                            # Spot
                             df.append(
                                 fxcf.get_fx_cross(start=md_request.start_date, end=md_request.finish_date, cross=t,
                                                   cut=md_request.cut, data_source=md_request.data_source,
@@ -151,21 +152,23 @@ def fetch_market(self, md_request=None):
                                                   environment=md_request.environment,
                                                   fields=['close']))
 
+                            # Entire FX vol surface
                             df.append(
-                                fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor,
+                                fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor_default,
                                                         cut=md_request.cut, data_source=md_request.data_source,
-                                                        part=fxvf.part,
+                                                        part=fxvf.part_default,
                                                         cache_algo=md_request.cache_algo))
 
+                            # FX forward points for every point on curve
                             df.append(rates.get_fx_forward_points(md_request.start_date, md_request.finish_date, t,
-                                                                  fxvf.tenor,
+                                                                  fxvf.tenor_default,
                                                                   cut=md_request.cut,
                                                                   data_source=md_request.data_source,
                                                                   cache_algo=md_request.cache_algo))
 
                     # Lastly fetch the base depos
                     df.append(rates.get_base_depos(md_request.start_date, md_request.finish_date,
-                                                   ["USD", "EUR", "CHF", "GBP"], fxvf.tenor,
+                                                   ["USD", "EUR", "CHF", "GBP"], fxvf.tenor_default,
                                                    cut=md_request.cut, data_source=md_request.data_source,
                                                    cache_algo=md_request.cache_algo
                                                    ))
@@ -212,7 +215,6 @@ class FXCrossFactory(object):
     """
 
     def __init__(self, market_data_generator=None):
-        self.logger = LoggerManager().getLogger(__name__)
         self.fxconv = FXConv()
 
         self.cache = {}
@@ -331,7 +333,7 @@ def get_fx_cross(self, start, end, cross,
         if market_data_request_list[0].data_source in constants.market_thread_no:
             thread_no = constants.market_thread_no[market_data_request_list[0].data_source]
 
-        # fudge, issue with multithreading and accessing HDF5 files
+        # Fudge, issue with multithreading and accessing HDF5 files
         # if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
         #    thread_no = 0
         thread_no = 0
@@ -477,7 +479,7 @@ def _get_individual_fx_cross(self, market_data_request):
                 cross_vals.columns = [cr + '-tot.close']
 
             elif freq == 'intraday':
-                self.logger.info('Total calculated returns for intraday not implemented yet')
+                LoggerManager().getLogger(__name__).info('Total calculated returns for intraday not implemented yet')
                 return None
 
         return cross_vals
@@ -491,20 +493,18 @@ def _get_individual_fx_cross(self, market_data_request):
 from findatapy.util import LoggerManager
 from findatapy.timeseries import Calculations, Filter, Timezone
 
-
 class FXVolFactory(object):
     """Generates FX implied volatility time series and surfaces (using very simple interpolation!) and only in delta space.
 
     """
     # types of quotation on vol surface
     # ATM, 25d riskies, 10d riskies, 25d strangles, 10d strangles
-    part = ["V", "25R", "10R", "25B", "10B"]
+    part_default = ["V", "25R", "10R", "25B", "10B"]
 
     # All the tenors on our vol surface
-    tenor = ["ON", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
+    tenor_default = ["ON", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
 
     def __init__(self, market_data_generator=None):
-        self.logger = LoggerManager().getLogger(__name__)
 
         self.market_data_generator = market_data_generator
 
@@ -517,8 +517,12 @@ def __init__(self, market_data_generator=None):
         return
 
     def get_fx_implied_vol(self, start, end, cross, tenor, cut="BGN", data_source="bloomberg", part="V",
-                           cache_algo="internet_load_return"):
-        """Get implied vol for specified cross, tenor and part of surface
+                           cache_algo="internet_load_return", environment='backtest'):
+        """Get implied vol for specified cross, tenor and part of surface. By default we use Bloomberg, but we could
+        use any data provider for which we have vol tickers.
+
+        Note, that for Bloomberg not every point will be quoted for each dataset (typically, BGN will have more points
+        than for example LDN)
 
         Parameters
         ----------
@@ -544,6 +548,12 @@ def get_fx_implied_vol(self, start, end, cross, tenor, cut="BGN", data_source="b
 
         market_data_generator = self.market_data_generator
 
+        if tenor is None:
+            tenor = self.tenor_default
+
+        if part is None:
+            part = self.part_default
+
         tickers = self.get_labels(cross, part, tenor)
 
         market_data_request = MarketDataRequest(
@@ -555,11 +565,60 @@ def get_fx_implied_vol(self, start, end, cross, tenor, cut="BGN", data_source="b
             tickers=tickers,
             fields=['close'],
             cache_algo=cache_algo,
-            environment='backtest'
+            environment=environment
         )
 
         data_frame = market_data_generator.fetch_market_data(market_data_request)
-        data_frame.index.name = 'Date'
+        # data_frame.index.name = 'Date'
+
+        # Special case for 10AM NYC cut
+        # - get some historical 10AM NYC data (only available on BBG for a few years, before 2007)
+        # - fill the rest with a weighted average of TOK/LDN closes
+        if cut == "10AM":
+            # Where we have actual 10am NY data use that & overwrite earlier estimated data (next)
+            vol_data_10am = data_frame
+
+            # As for most dates we probably won't have 10am data
+            if vol_data_10am is not None:
+                vol_data_10am = vol_data_10am.dropna()  # Only have limited ON 10am cut data
+
+            # Now get LDN and TOK vol data to fill any gaps
+            vol_data_LDN = self.get_fx_implied_vol(start=start, end=end, cross=cross, tenor=tenor,
+                                                                       data_source=data_source, cut='LDN', part=part,
+                                                                       cache_algo=cache_algo)
+
+            vol_data_TOK = self.get_fx_implied_vol(start=start, end=end, cross=cross, tenor=tenor,
+                                                                       data_source=data_source, cut='TOK', part=part,
+                                                                       cache_algo=cache_algo)
+
+            # vol_data_LDN.index = pandas.DatetimeIndex(vol_data_LDN.index)
+            # vol_data_TOK.index = pandas.DatetimeIndex(vol_data_TOK.index)
+
+            old_cols = vol_data_LDN.columns
+
+            vol_data_LDN.columns = vol_data_LDN.columns.values + "LDN"
+            vol_data_TOK.columns = vol_data_TOK.columns.values + "TOK"
+
+            data_frame = vol_data_LDN.join(vol_data_TOK, how='outer')
+
+            # Create very naive average of LDN and TOK to estimate 10am NY value because we often don't have this data
+            # Note, this isn't perfect, particularly on days where you have payrolls data, and we're looking at ON data
+            for col in old_cols:
+                data_frame[col] = (1 * data_frame[col + "LDN"] + 3 * data_frame[col + "TOK"]) / 4
+
+                data_frame.pop(col + "LDN")
+                data_frame.pop(col + "TOK")
+
+            # Get TOK/LDN vol data before 10am and after 10am (10am data is only available for a few years)
+            # If we have no original 10am data don't bother
+            if vol_data_10am is not None:
+                if not(vol_data_10am.empty):
+                    pre_vol_data = data_frame[data_frame.index < vol_data_10am.index[0]]
+                    post_vol_data = data_frame[data_frame.index > vol_data_10am.index[-1]]
+
+                    data_frame = (pre_vol_data.append(vol_data_10am)).append(post_vol_data)
+
+            # data_frame.index = pandas.to_datetime(data_frame.index)
 
         return data_frame
 
@@ -578,6 +637,21 @@ def get_labels(self, cross, part, tenor):
         return tickers
 
     def extract_vol_surface_for_date(self, df, cross, date_index):
+        """Get's the vol surface in delta space without any interpolation
+
+        Parameters
+        ----------
+        df : DataFrame
+            With vol data
+        cross : str
+            Currency pair
+        date_index : int
+            Which date to extract
+
+        Returns
+        -------
+        DataFrame
+        """
 
         # Assume we have a matrix of the form
         # eg. EURUSDVON.close ...
@@ -594,7 +668,7 @@ def extract_vol_surface_for_date(self, df, cross, date_index):
                    "25DC",
                    "10DC"]
 
-        tenor = self.tenor
+        tenor = self.tenor_default
 
         df_surf = pandas.DataFrame(index=strikes, columns=tenor)
 
@@ -623,12 +697,11 @@ def extract_vol_surface_for_date(self, df, cross, date_index):
 #######################################################################################################################
 
 class RatesFactory(object):
-    """Gets the deposit rates for a particular currency
+    """Gets the deposit rates for a particular currency (or forwards for a currency pair)
 
     """
 
     def __init__(self, market_data_generator=None):
-        self.logger = LoggerManager().getLogger(__name__)
 
         self.cache = {}
 
@@ -669,6 +742,9 @@ def get_base_depos(self, start, end, currencies, tenor, cut="NYC", data_source="
 
         market_data_generator = self.market_data_generator
 
+        if tenor is None:
+            tenor = FXVolFactory.tenor_default
+
         if isinstance(currencies, str): currencies = [currencies]
         if isinstance(tenor, str): tenor = [tenor]
 
@@ -722,16 +798,20 @@ def get_fx_forward_points(self, start, end, cross, tenor, cut="BGN", data_source
         Contains deposit rates
         """
 
-        market_data_request = MarketDataRequest()
+        # market_data_request = MarketDataRequest()
         market_data_generator = self.market_data_generator
 
-        market_data_request.data_source = data_source  # use bbg as a data_source
-        market_data_request.start_date = start  # start_date
-        market_data_request.finish_date = end  # finish_date
+        # market_data_request.data_source = data_source  # use bbg as a data_source
+        # market_data_request.start_date = start  # start_date
+        # market_data_request.finish_date = end  # finish_date
+
+        if tenor is None:
+            tenor = FXVolFactory.tenor_default
 
         if isinstance(cross, str): cross = [cross]
         if isinstance(tenor, str): tenor = [tenor]
 
+        # Tickers are often different on Bloomberg for forwards/depos vs vol, so want consistency
         tenor = [x.replace('1Y', '12M') for x in tenor]
 
         tickers = []

findatapy/market/marketdatarequest.py

@@ -84,7 +84,7 @@ def __init__(self, data_source=None,
                  overrides={}
                  ):
 
-        # can deep copy MarketDataRequest (use a lock, so can be used with threading when downloading time series)
+        # Can deep copy MarketDataRequest (use a lock, so can be used with threading when downloading time series)
         if md_request is not None:
             import threading
             lock = threading.Lock()
@@ -126,7 +126,7 @@ def __init__(self, data_source=None,
                 self.overrides = copy.deepcopy(md_request.overrides)
                 self.push_to_cache = copy.deepcopy(md_request.push_to_cache)
 
-                self.tickers = copy.deepcopy(md_request.tickers)  # need this after category in case have wildcard
+                self.tickers = copy.deepcopy(md_request.tickers)  # Need this after category in case have wildcard
         else:
             self.freq_mult = freq_mult