Fixed delete key problem with Redis

cuemacro · Jan 25, 2022 · 5f619b3 · 5f619b3
1 parent 6c18cd1
commit 5f619b3
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -61,8 +61,6 @@ Major requirements
 * Recommended: blpapi - Bloomberg Python Open API
 * Recommended: chartpy - for funky interactive plots ([https://github.com/cuemacro/chartpy](https://github.com/cuemacro/chartpy)) and
 * Recommended: arctic - AHL library for managing time series in MongoDB
-* Recommended: multiprocessor_on_dill - standard multiprocessing library pickle causes issues ([https://github.com/sixty-north/multiprocessing_on_dill](https://github.com/sixty-north/multiprocessing_on_dill))
-* Recommended: fredapi - ALFRED/FRED has been rewritten and added to the project directly (from https://github.com/mortada/fredapi)
 
 # Installation
 
@@ -135,6 +133,8 @@ individual data providers)
 
 # Coding log
 
+* 25 Jan 2022
+  * Fixed delete key problem with Redis
 * 20 Jan 2022
   * Fixed path join for s3
 * 17 Jan 2022

diff --git a/findatapy/market/ioengine.py b/findatapy/market/ioengine.py
@@ -172,10 +172,7 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed',
         if 'hdf5' in engine:
             engine = 'hdf5'
 
-        if (engine == 'bcolz'):
-            # convert invalid characters to substitutes (which Bcolz can't deal with)
-            pass
-        elif (engine == 'redis'):
+        if engine == 'redis':
 
             fname = os.path.basename(fname).replace('.', '_')
 
@@ -184,15 +181,20 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed',
                                       socket_timeout=timeout,
                                       socket_connect_timeout=timeout)
 
-                if (fname == 'flush_all_keys'):
+                if fname == 'flush_all_keys':
                     r.flushall()
                 else:
-                    # allow deletion of keys by pattern matching
+                    # Allow deletion of keys by pattern matching
+                    matching_keys = r.keys('*' + fname)
 
-                    x = r.keys('*' + fname)
+                    if matching_keys:
+                        # Use pipeline to speed up command
+                        pipe = r.pipeline()
 
-                    if len(x) > 0:
-                        r.delete(x)
+                        for key in matching_keys:
+                            pipe.delete(key)
+
+                        pipe.execute()
 
                     # r.delete(fname)
 
@@ -230,7 +232,7 @@ def remove_time_series_cache_on_disk(self, fname, engine='hdf5_fixed',
 
             logger.info("Deleted MongoDB library: " + fname)
 
-        elif (engine == 'hdf5'):
+        elif engine == 'hdf5':
             h5_filename = self.get_h5_filename(fname)
 
             # delete the old copy

diff --git a/findatapy/timeseries/calendar.py b/findatapy/timeseries/calendar.py
@@ -32,6 +32,8 @@
 
 constants = DataConstants()
 
+# To speed up CustomBusinessDay
+# https://stackoverflow.com/questions/31523302/performance-of-pandas-custom-business-day-offset
 
 class Calendar(object):
     """Provides calendar based functions for working out options expiries,