Fixed Calendar for single Timestamps

README.md

@@ -125,6 +125,9 @@ individual data providers)
 
 # Coding log
 
+* 24 Dec 2020
+  * Remove logger as field variable in IOEngine
+  * Fixed Calendar methods so can take single input
 * 19 Dec 2020
   * Added functionality to download FX forwards based total indices from BBG
   * Fixed downloading of forward points for NDFs

findatapy/conf/base_depos_tickers_list.csv

@@ -1,5 +1,5 @@
 category,source,freq,ticker,cut,fields,sourceticker,local-close
-base-depos,bloomberg,daily,Fed Funds Effective Rate,NYC,close,FEDL01 Index,5:00pm NYC
+base-depos,bloomberg,daily,USDFedEffectiveRate,NYC,close,FEDL01 Index,5:00pm NYC
 base-depos,bloomberg,daily,USDON,NYC,close,USDR1T CMPN Index,5:00pm NYC
 base-depos,bloomberg,daily,USDTN,NYC,close,USDR2T CMPN Index,5:00pm NYC
 base-depos,bloomberg,daily,USDSN,NYC,close,USDR3T CMPN Index,5:00pm NYC

findatapy/market/ioengine.py

@@ -74,7 +74,7 @@ class IOEngine(object):
     """
 
     def __init__(self):
-        self.logger = LoggerManager().getLogger(__name__)
+        pass
 
     ### functions to handle Excel on disk
     def write_time_series_to_excel(self, fname, sheet, data_frame, create_new=False):
@@ -153,6 +153,8 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed', db_server
                                          timeout=10, username=None,
                                          password=None):
 
+        logger = LoggerManager().getLogger(__name__)
+
         if 'hdf5' in engine:
             engine = 'hdf5'
 
@@ -180,7 +182,7 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed', db_server
                     # r.delete(fname)
 
             except Exception as e:
-                self.logger.warning("Cannot delete non-existent key " + fname + " in Redis: " + str(e))
+                logger.warning("Cannot delete non-existent key " + fname + " in Redis: " + str(e))
 
         elif (engine == 'arctic'):
             from arctic import Arctic
@@ -189,7 +191,7 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed', db_server
             socketTimeoutMS = 30 * 1000
             fname = os.path.basename(fname).replace('.', '_')
 
-            self.logger.info('Load MongoDB library: ' + fname)
+            logger.info('Load MongoDB library: ' + fname)
 
             if username is not None and password is not None:
                 c = pymongo.MongoClient(
@@ -205,7 +207,7 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed', db_server
 
             c.close()
 
-            self.logger.info("Deleted MongoDB library: " + fname)
+            logger.info("Deleted MongoDB library: " + fname)
 
         elif (engine == 'hdf5'):
             h5_filename = self.get_h5_filename(fname)
@@ -246,6 +248,8 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
             Number of seconds to do timeout
         """
 
+        logger = LoggerManager().getLogger(__name__)
+
         # default HDF5 format
         hdf5_format = 'fixed'
 
@@ -289,12 +293,12 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
                         else:
                             r.set(fname, ser.to_pybytes())
 
-                    self.logger.info("Pushed " + fname + " to Redis")
+                    logger.info("Pushed " + fname + " to Redis")
                 else:
-                    self.logger.info("Object " + fname + " is empty, not pushed to Redis.")
+                    logger.info("Object " + fname + " is empty, not pushed to Redis.")
 
             except Exception as e:
-                self.logger.warning("Couldn't push " + fname + " to Redis: " + str(e))
+                logger.warning("Couldn't push " + fname + " to Redis: " + str(e))
 
         elif (engine == 'arctic'):
             from arctic import Arctic
@@ -303,7 +307,7 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
             socketTimeoutMS = 30 * 1000
             fname = os.path.basename(fname).replace('.', '_')
 
-            self.logger.info('Load Arctic/MongoDB library: ' + fname)
+            logger.info('Load Arctic/MongoDB library: ' + fname)
 
             if username is not None and password is not None:
                 c = pymongo.MongoClient(
@@ -324,9 +328,9 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
 
             if database is None:
                 store.initialize_library(fname, audit=False)
-                self.logger.info("Created MongoDB library: " + fname)
+                logger.info("Created MongoDB library: " + fname)
             else:
-                self.logger.info("Got MongoDB library: " + fname)
+                logger.info("Got MongoDB library: " + fname)
 
             # Access the library
             library = store[fname]
@@ -356,7 +360,7 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
 
             c.close()
 
-            self.logger.info("Written MongoDB library: " + fname)
+            logger.info("Written MongoDB library: " + fname)
 
         elif (engine == 'hdf5'):
             h5_filename = self.get_h5_filename(fname)
@@ -413,7 +417,7 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
                 # once written to disk rename
                 os.rename(h5_filename_temp, h5_filename)
 
-            self.logger.info("Written HDF5: " + fname)
+            logger.info("Written HDF5: " + fname)
 
         elif (engine == 'parquet'):
             if '.parquet' not in fname:
@@ -422,7 +426,7 @@ def write_time_series_cache_to_disk(self, fname, data_frame,
 
             data_frame.to_parquet(fname, compression=parquet_compression)
 
-            self.logger.info("Written Parquet: " + fname)
+            logger.info("Written Parquet: " + fname)
 
     def get_h5_filename(self, fname):
         """Strips h5 off filename returning first portion of filename
@@ -470,9 +474,12 @@ def write_r_compatible_hdf_dataframe(self, data_frame, fname, fields=None):
         fields : list(str)
             columns to be written
         """
+
+        logger = LoggerManager().getLogger(__name__)
+
         fname_r = self.get_h5_filename(fname)
 
-        self.logger.info("About to dump R binary HDF5 - " + fname_r)
+        logger.info("About to dump R binary HDF5 - " + fname_r)
         data_frame32 = data_frame.astype('float32')
 
         if fields is None:
@@ -581,15 +588,15 @@ def read_time_series_cache_from_disk(self, fname, engine='hdf5', start_date=None
                         # print(fname_single)
                         if msg is not None:
                             msg = context.deserialize(msg)
-                            # self.logger.warning("Key " + fname_single + " not in Redis cache?")
+                            # logger.warning("Key " + fname_single + " not in Redis cache?")
 
                 except Exception as e:
-                    self.logger.info("Cache not existent for " + fname_single + " in Redis: " + str(e))
+                    logger.info("Cache not existent for " + fname_single + " in Redis: " + str(e))
 
                 if msg is None:
                     data_frame = None
                 else:
-                    self.logger.info('Load Redis cache: ' + fname_single)
+                    logger.info('Load Redis cache: ' + fname_single)
 
                     data_frame = msg # pandas.read_msgpack(msg)
 
@@ -601,7 +608,7 @@ def read_time_series_cache_from_disk(self, fname, engine='hdf5', start_date=None
 
                 fname_single = os.path.basename(fname_single).replace('.', '_')
 
-                self.logger.info('Load Arctic/MongoDB library: ' + fname_single)
+                logger.info('Load Arctic/MongoDB library: ' + fname_single)
 
                 if username is not None and password is not None:
                     c = pymongo.MongoClient(
@@ -625,12 +632,12 @@ def read_time_series_cache_from_disk(self, fname, engine='hdf5', start_date=None
 
                     c.close()
 
-                    self.logger.info('Read ' + fname_single)
+                    logger.info('Read ' + fname_single)
 
                     data_frame = item.data
 
                 except Exception as e:
-                    self.logger.warning('Library may not exist or another error: ' + fname_single + ' & message is ' + str(e))
+                    logger.warning('Library may not exist or another error: ' + fname_single + ' & message is ' + str(e))
                     data_frame = None
 
             elif os.path.isfile(self.get_h5_filename(fname_single)):
@@ -779,7 +786,9 @@ def convert_csv_data_frame(self, f_name, category, freq, cutoff=None, dateparse=
             date parser to use
         """
 
-        self.logger.info("About to read... " + f_name)
+        logger = LoggerManager().getLogger(__name__)
+
+        logger.info("About to read... " + f_name)
 
         data_frame = self.read_csv_data_frame(f_name, freq, cutoff=cutoff, dateparse=dateparse)
 
@@ -795,13 +804,14 @@ def clean_csv_file(self, f_name):
         f_name : str
             CSV file to be cleaned
         """
+        logger = LoggerManager().getLogger(__name__)
 
         with codecs.open(f_name, 'rb', 'utf-8') as myfile:
             data = myfile.read()
 
             # clean file first if dirty
             if data.count('\x00'):
-                self.logger.info('Cleaning CSV...')
+                logger.info('Cleaning CSV...')
 
                 with codecs.open(f_name + '.tmp', 'w', 'utf-8') as of:
                     of.write(data.replace('\x00', ''))

findatapy/timeseries/filter.py

@@ -1,7 +1,7 @@
 __author__ = 'saeedamen'  # Saeed Amen
 
 #
-# Copyright 2016 Cuemacro
+# Copyright 2016-2020 Cuemacro
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
 # License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
@@ -25,7 +25,8 @@
 
 import pandas.tseries.offsets
 
-from pandas.tseries.offsets import BDay, CustomBusinessDay, Day, CustomBusinessMonthEnd, MonthOffset
+from pandas.tseries.offsets import BDay, CustomBusinessDay, Day, CustomBusinessMonthEnd, DateOffset
+
 
 from findatapy.timeseries.timezone import Timezone
 
@@ -196,8 +197,8 @@ def filter_time_series_by_holidays(self, data_frame, cal='FX', holidays_list=[])
         DataFrame
         """
 
-        # optimal case for weekdays: remove Saturday and Sunday
-        if (cal == 'WEEKDAY'):
+        # Optimal case for weekdays: remove Saturday and Sunday
+        if (cal == 'WEEKDAY' or cal == 'WKY'):
             return data_frame[data_frame.index.dayofweek <= 4]
 
         # Select only those holidays in the sample
@@ -899,6 +900,10 @@ def get_dates_from_tenors(self, start, end, tenor, cal=None):
         freq = str(self.get_business_days_tenor(tenor)) + "B"
         return pd.DataFrame(index=pd.bdate_range(start, end, freq=freq))
 
+    def get_delta_between_dates(self, date1, date2, unit='days'):
+        if unit == 'days':
+            return (date2 - date1).days
+
     def get_delivery_date_from_horizon_date(self, horizon_date, tenor, cal=None, asset_class='fx'):
         if 'fx' in asset_class:
             tenor_unit = ''.join(re.compile(r'\D+').findall(tenor))
@@ -929,12 +934,18 @@ def get_delivery_date_from_horizon_date(self, horizon_date, tenor, cal=None, ass
                     tenor_digit = tenor_digit * 12
 
                 horizon_period_end = horizon_date + CustomBusinessMonthEnd(tenor_digit + 1)
-                horizon_floating = horizon_date + MonthOffset(tenor_digit)
+                horizon_floating = horizon_date + DateOffset(months=tenor_digit)
 
                 cbd = CustomBusinessDay(n=1, holidays=asset_holidays)
 
                 delivery_date = []
 
+                if isinstance(horizon_period_end, pd.Timestamp):
+                    horizon_period_end = [horizon_period_end]
+
+                if isinstance(horizon_floating, pd.Timestamp):
+                    horizon_floating = [horizon_floating]
+
                 for period_end, floating in zip(horizon_period_end, horizon_floating):
                     if floating < period_end:
                         delivery_date.append(floating - cbd + cbd)
@@ -991,12 +1002,18 @@ def get_expiry_date_from_horizon_date(self, horizon_date, tenor, cal=None, asset
                     tenor_digit = tenor_digit * 12
 
                 horizon_period_end = horizon_date + CustomBusinessMonthEnd(tenor_digit + 1)
-                horizon_floating = horizon_date + MonthOffset(tenor_digit)
+                horizon_floating = horizon_date + DateOffset(months=tenor_digit)
 
                 cbd = CustomBusinessDay(n=1, holidays=asset_holidays)
 
                 delivery_date = []
 
+                if isinstance(horizon_period_end, pd.Timestamp):
+                    horizon_period_end = [horizon_period_end]
+
+                if isinstance(horizon_floating, pd.Timestamp):
+                    horizon_floating = [horizon_floating]
+
                 for period_end, floating in zip(horizon_period_end, horizon_floating):
                     if floating < period_end:
                         delivery_date.append(floating - cbd + cbd)

findatapy/util/dataconstants.py

@@ -173,15 +173,16 @@ class DataConstants(object):
     # All the tenors on our vol surface
     fx_vol_tenor = ["ON", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
 
-    # All the tenors on our forwards
-    fx_forwards_tenor = ["ON", "TN", "SN", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
-
     # Which base depo currencies are available?
     base_depos_currencies = ['EUR', 'GBP', 'AUD', 'NZD', 'USD', 'CAD', 'CHF', 'NOK', 'SEK', 'JPY']
 
     # Tenors available for base depos
     base_depos_tenor = ["ON", "TN", "SN", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
 
+    ### FX forwards total return index construction
+    # All the tenors on our forwards
+    fx_forwards_tenor = ["ON", "TN", "SN", "1W", "2W", "3W", "1M", "2M", "3M", "4M", "6M", "9M", "1Y", "2Y", "3Y", "5Y"]
+
     # overwrite field variables with those listed in DataCred
     def __init__(self):
         try:

findatapy/util/tickerfactory.py

@@ -95,7 +95,7 @@ def create_ticker(self, csv_file, out_csv_file):
     tf = TickerFactory()
 
     root = 'E:/cuemacro/yen/conf'
-    root = 'E:/cuemacro/findatapy/findatapy/conf'
+    # root = 'E:/cuemacro/findatapy/findatapy/conf'
 
     csv_file = root + '/fx_vol_tickers_maker.csv'
     out_csv_file = root + '/fx_vol_tickers.csv'

findatapy_examples/calendar_examples.py

@@ -28,8 +28,9 @@
     # choose run_example = 0 for everything
     # run_example = 1 - get holidays for FX, EUR and EURUSD, as well as listing weekends
     # run_example = 2 - get FX delivery dates and FX option expiries for various tenors
+    # run_example = 3 - get number of days between pandas DatetimeIndex
 
-    run_example = 0
+    run_example = 3
 
     if run_example == 1 or run_example == 0:
 
@@ -57,4 +58,12 @@
         print(calendar.get_expiry_date_from_horizon_date(
             pd.to_datetime([pd.Timestamp('26 Oct 2020')]), '1M', cal='EURUSD'))
 
+    if run_example == 3 or run_example == 0:
+        # Create a list of business days and one which is + 1 day
+
+        bus_days = pd.bdate_range('1 Jan 2020', '30 Jan 2020')
+        bus_days_plus = bus_days + pd.Timedelta(days=7)
+
+        print(calendar.get_delta_between_dates(bus_days, bus_days_plus))
+