More pep8 newline and whitespace fixes

Only fixing small whitespace, extra or missing newline pep8 issues:

Basically involves a run off:
autopep8 -ir --max-line-length=500 --select E201,W391,W293,E303,E265,E305,W292,E231  .

For all other issues, will wait till we can discuss possibly using
an auto formatter with a possible pylint/pep8 check in travis.

arctic/__init__.py

@@ -26,4 +26,3 @@
 register_versioned_storage(PandasSeriesStore)
 register_versioned_storage(PandasPanelStore)
 register_versioned_storage(NdarrayStore)
-

arctic/_compression.py

@@ -86,7 +86,7 @@ def compress_array(str_list, withHC=LZ4_HIGH_COMPRESSION):
         return _compress_thread_pool.map(do_compress, str_list)
 
     return [do_compress(s) for s in str_list]
-    
+
 
 def compress(_str):
     """

arctic/_config.py

@@ -43,6 +43,7 @@ class FwPointersCfg(Enum):
     DISABLED = 1  # operate in legacy mode, update segment parent references, don't add forward pointers
     HYBRID = 2    # maintain both forward pointers and parent references in segments; for reads prefer fw pointers
 
+
 # The version document key used to store the ObjectIDs of segments
 FW_POINTERS_REFS_KEY = 'SEGMENT_SHAS'
 

arctic/arctic.py

@@ -229,7 +229,6 @@ def library_exists(self, library):
             pass
         return exists
 
-
     @mongo_retry
     def initialize_library(self, library, lib_type=VERSION_STORE, **kwargs):
         """
@@ -484,7 +483,7 @@ def __setstate__(self, state):
 
     @mongo_retry
     def _auth(self, database):
-        #Get .mongopass details here
+        # Get .mongopass details here
         if not hasattr(self.arctic, 'mongo_host'):
             return
 

arctic/chunkstore/chunkstore.py

@@ -233,7 +233,6 @@ def rename(self, from_symbol, to_symbol, audit=None):
             audit['old_symbol'] = from_symbol
             self._audit.insert_one(audit)
 
-
     def read(self, symbol, chunk_range=None, filter_data=True, **kwargs):
         """
         Reads data for a given symbol from the database.
@@ -264,7 +263,6 @@ def read(self, symbol, chunk_range=None, filter_data=True, **kwargs):
         if not sym:
             raise NoDataFoundException('No data found for %s' % (symbol))
 
-
         spec = {SYMBOL: {'$in': symbol}}
         chunker = CHUNKER_MAP[sym[0][CHUNKER]]
         deser = SER_MAP[sym[0][SERIALIZER]].deserialize
@@ -289,7 +287,6 @@ def read(self, symbol, chunk_range=None, filter_data=True, **kwargs):
             chunk_data = b''.join([doc[DATA] for doc in segments])
             chunks[segments[0][SYMBOL]].append({DATA: chunk_data, METADATA: mdata})
 
-
         skip_filter = not filter_data or chunk_range is None
 
         if len(symbol) > 1:
@@ -470,7 +467,6 @@ def __update(self, sym, item, metadata=None, combine_method=None, chunk_range=No
                                               END: end,
                                               SEGMENT: {'$gte': chunk_count}})
 
-
             for i in xrange(chunk_count):
                 chunk = {DATA: Binary(data[DATA][i * MAX_CHUNK_SIZE: (i + 1) * MAX_CHUNK_SIZE])}
                 chunk[SEGMENT] = i

arctic/chunkstore/tools/__init__.py

@@ -1 +1 @@
-from .tools import segment_id_repair
+from .tools import segment_id_repair

arctic/chunkstore/tools/tools.py

@@ -45,5 +45,5 @@ def segment_id_repair(library, symbol=None):
                     seg[SEGMENT] = index
                 library._collection.insert_many(segments)
                 ret.append(sym)
-    
-    return ret
+
+    return ret

arctic/date/_generalslice.py

@@ -3,6 +3,8 @@
 
 class Intervals(Enum):
     (OPEN_CLOSED, CLOSED_OPEN, OPEN_OPEN, CLOSED_CLOSED) = range(1101, 1105)
+
+
 (OPEN_CLOSED, CLOSED_OPEN, OPEN_OPEN, CLOSED_CLOSED) = INTERVALS = Intervals.__members__.values()
 
 

arctic/date/_util.py

@@ -11,7 +11,7 @@
 if sys.version_info > (3,):
     long = int
 
-    
+
 # Support standard brackets syntax for open/closed ranges.
 Ranges = {'()': OPEN_OPEN,
           '(]': OPEN_CLOSED,

arctic/decorators.py

@@ -26,6 +26,7 @@ def _get_host(store):
             pass
     return ret
 
+
 _in_retry = False
 _retry_count = 0
 

arctic/scripts/arctic_fsck.py

@@ -70,5 +70,6 @@ def main():
     else:
         logger.info("Done.")
 
+
 if __name__ == '__main__':
     main()

arctic/serialization/incremental.py

@@ -84,7 +84,7 @@ def _dtype_convert_to_max_len_string(self, input_ndtype, fname):
         max_str_len = len(max(self.input_data[fname].astype(type_sym), key=len))
         str_field_dtype = np.dtype('{}{:d}'.format(type_sym, max_str_len)) if max_str_len > 0 else input_ndtype
         return str_field_dtype, True
-    
+
     def _get_dtype(self):
         # Serializer is being called only if can_convert_to_records_without_objects() has passed,
         # which means that the resulting recarray does not contain objects but only numpy types, string, or unicode

arctic/serialization/numpy_arrays.py

@@ -26,7 +26,6 @@
 from ._serializer import Serializer
 
 
-
 DATA = 'd'
 MASK = 'm'
 TYPE = 't'

arctic/serialization/numpy_records.py

@@ -21,7 +21,6 @@
 DTN64_DTYPE = 'datetime64[ns]'
 
 
-
 def set_fast_check_df_serializable(config):
     global FAST_CHECK_DF_SERIALIZABLE
     FAST_CHECK_DF_SERIALIZABLE = bool(config)
@@ -188,7 +187,7 @@ def fast_check_serializable(self, df):
     def can_convert_to_records_without_objects(self, df, symbol):
         # We can't easily distinguish string columns from objects
         try:
-            #TODO: we can add here instead a check based on df size and enable fast-check if sz > threshold value
+            # TODO: we can add here instead a check based on df size and enable fast-check if sz > threshold value
             if FAST_CHECK_DF_SERIALIZABLE:
                 arr, _ = self.fast_check_serializable(df)
             else:

arctic/store/_ndarray_store.py

@@ -396,7 +396,7 @@ def append(self, arctic_lib, version, symbol, item, previous_version, dtype=None
 
         if (self._dtype(previous_version['dtype']).fields is None) != (dtype.fields is None):
             raise ValueError("type changes to or from structured array not supported")
-        
+
         if previous_version['up_to'] == 0:
             dtype = dtype
         elif len(item) == 0:
@@ -429,7 +429,7 @@ def append(self, arctic_lib, version, symbol, item, previous_version, dtype=None
         else:
             version['dtype'] = previous_version['dtype']
             version['dtype_metadata'] = previous_version['dtype_metadata']
-            
+
             # Verify (potential) corruption with append
             if CHECK_CORRUPTION_ON_APPEND and _fast_check_corruption(
                     collection, symbol, previous_version,
@@ -454,8 +454,8 @@ def _do_append(self, collection, version, symbol, item, previous_version, dirty_
             version['append_count'] = previous_version['append_count']
             version['append_size'] = previous_version['append_size']
 
-        #_CHUNK_SIZE is probably too big if we're only appending single rows of data - perhaps something smaller,
-        #or also look at number of appended segments?
+        # _CHUNK_SIZE is probably too big if we're only appending single rows of data - perhaps something smaller,
+        # or also look at number of appended segments?
         if not dirty_append and version['append_count'] < _APPEND_COUNT and version['append_size'] < _APPEND_SIZE:
             version['base_version_id'] = version_base_or_id(previous_version)
 

arctic/store/_version_store_utils.py

@@ -247,15 +247,15 @@ def _fast_check_corruption(collection, sym, v, check_count, check_last_segment,
     if v is None:
         logging.warning("Symbol {} with version {} not found, so can't be corrupted.".format(sym, v))
         return False
-    
+
     if not check_count and not check_last_segment:
         raise ValueError("_fast_check_corruption must be called with either of "
                          "check_count and check_last_segment set to True")
 
     # If version marked symbol as deleted, it will force writes/appends to start from a new base: non corrupted.
     if isinstance(v.get('metadata'), dict) and v['metadata'].get('deleted'):
         return False
-     
+
     if check_append_safe:
         # Check whether appending to the symbol version can potentially corrupt the data (history branch).
         # Inspect all segments, don't limit to v['up_to']. No newer append segments after v should exist.
@@ -280,7 +280,7 @@ def _fast_check_corruption(collection, sym, v, check_count, check_last_segment,
             # Quick check: Segment counts agree and size is zero
             if total_segments == 0:
                 return False
-        
+
         if check_last_segment:
             # Quick check: compare the maximum segment's up_to number. It has to verify the version's up_to.
             max_seg = collection.find_one(spec, {'segment': 1}, sort=[('segment', pymongo.DESCENDING)])
@@ -373,9 +373,6 @@ def is_corrupted(l, sym, input_v):
     return True
 
 
-
-
-
 # Initialise the pickle load function and delete the factory function.
 pickle_compat_load = _define_compat_pickle_load()
 del _define_compat_pickle_load

arctic/store/audit.py

@@ -94,7 +94,7 @@ def __init__(self, version_store, symbol, user, log, modify_timeseries=None, aud
             self.base_ts = VersionedItem(symbol=self._symbol, library=None,
                                          version=versions[0], metadata=None, data=None, host=None)
         except OperationFailure:
-            #TODO: Current errors in mongo "Incorrect Number of Segments Returned"
+            # TODO: Current errors in mongo "Incorrect Number of Segments Returned"
             # This workaround should be removed once underlying problem is resolved.
             self.base_ts = self._version_store.read_metadata(symbol=self._symbol)
 

arctic/store/bson_store.py

@@ -178,7 +178,7 @@ def distinct(self, key, **kwargs):
         See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.distinct
         """
         return self._collection.distinct(key, **kwargs)
-    
+
     @mongo_retry
     def create_index(self, keys, **kwargs):
         """

arctic/store/version_store.py

@@ -703,7 +703,7 @@ def _add_new_version_using_reference(self, symbol, new_version, reference_versio
         if lastv_seqn != new_version['version']:
             raise OperationFailure("The symbol {} has been modified concurrently ({} != {})".format(
                 symbol, lastv_seqn, new_version['version']))
-        
+
         # Insert the new version into the version DB
         # (must come before the pruning, otherwise base version won't be preserved)
         self._insert_version(new_version)

arctic/tickstore/tickstore.py

@@ -186,11 +186,11 @@ def _mongo_date_range_query(self, symbol, date_range):
         if date_range.start:
             assert date_range.start.tzinfo
             start = date_range.start
-            
+
             # If all chunks start inside of the range, we default to capping to our
             # range so that we don't fetch any chunks from the beginning of time
             start_range['$gte'] = start
-            
+
             match = self._symbol_query(symbol)
             match.update({'s': {'$lte': start}})
 
@@ -216,7 +216,6 @@ def _mongo_date_range_query(self, symbol, date_range):
                         break
             except StopIteration:
                 pass
-
 
         # Find the end bound
         if date_range.end:

benchmarks/__init__.py

@@ -1 +0,0 @@
-

benchmarks/benchmarks.py

@@ -46,13 +46,13 @@ def gen_series_compressible(rows):
 class TimeSuiteWrite(object):
     params = list(range(len(TEST_SIZES)))
     param_names = ['5K * 10^']
-    
+
     def setup(self, arg):
         self.store = Arctic("127.0.0.1")
         self.store.delete_library('test.lib')
         self.store.initialize_library('test.lib')
         self.lib = self.store['test.lib']
-        
+
     def teardown(self, arg):
         self.store.delete_library('test.lib')
         self.lib = None

howtos/201507_demo_pydata.py

@@ -38,7 +38,6 @@ def get_stock_history(ticker, start_date, end_date):
 ################################################
 
 
-
 arctic = Arctic('localhost')
 
 # Create a VersionStore library
@@ -80,13 +79,12 @@ def get_stock_history(ticker, start_date, end_date):
 stocks.read('aapl', as_of='snap').data.ix[-1]
 
 
-
 #################################
 # Dealing with lots of data
 #################################
 
 
-#NSYE library
+# NSYE library
 lib = arctic['nyse']
 
 def load_all_stock_history_NYSE():

howtos/how_to_custom_arctic_library.py

@@ -138,7 +138,7 @@ def delete(self, query):
     mongo_host = 'localhost'
 store = Arctic(mongo_host)
 
-### Initialize the library
+# Initialize the library
 # Map username.custom_lib -> CustomArcticLibType
 store.initialize_library('username.custom_lib', CustomArcticLibType._LIBRARY_TYPE)
 
@@ -165,7 +165,7 @@ def delete(self, query):
 list(lib.query({'field1': 'a'}))     # Can query lists
 list(lib.query({'field1': 'b'}))
 list(lib.query({'date_field': {'$lt': dt(2013, 2, 2)}}))
-list(lib.query({'field1':'thing',
+list(lib.query({'field1': 'thing',
                   'date_field': {'$lt': dt(2013, 2, 2)} }))
 
 # Remove everything