Skip to content

Commit

Permalink
Fix issue 216 (pandas-dev#321)
Browse files Browse the repository at this point in the history
Addresses issue 216 by allowing index column to be named in tickstore
  • Loading branch information
bmoscon committed Jan 30, 2017
1 parent c0fc4d0 commit 43840f7
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 4 deletions.
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
* Bugfix: #300 to_datetime deprecated in pandas, use to_pydatetime instead
* Bugfix: #309 formatting change for DateRange ```__str__```
* Feature: #313 set and read user specified metadata in chunkstore
* Bugfix: #216 Tickstore write fails with named index column

### 1.36 (2016-12-13)

Expand Down
9 changes: 5 additions & 4 deletions arctic/tickstore/tickstore.py
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,8 @@ def write(self, symbol, data, initial_image=None):
data : list of dicts or a pandas.DataFrame
List of ticks to store to the tick-store.
if a list of dicts, each dict must contain a 'index' datetime
if a pandas.DataFrame the index must be a Timestamp that can be converted to a datetime
if a pandas.DataFrame the index must be a Timestamp that can be converted to a datetime.
Index names will not be preserved.
initial_image : dict
Dict of the initial image at the start of the document. If this contains a 'index' entry it is
assumed to be the time of the timestamp of the index
Expand Down Expand Up @@ -628,6 +629,7 @@ def _pandas_to_bucket(df, symbol, initial_image):
logger.warning("NB treating all values as 'exists' - no longer sparse")
rowmask = Binary(lz4.compressHC(np.packbits(np.ones(len(df), dtype='uint8'))))

index_name = df.index.names[0] or "index"
recs = df.to_records(convert_datetime64=False)
for col in df:
array = TickStore._ensure_supported_dtypes(recs[col])
Expand All @@ -636,9 +638,8 @@ def _pandas_to_bucket(df, symbol, initial_image):
col_data[ROWMASK] = rowmask
col_data[DTYPE] = TickStore._str_dtype(array.dtype)
rtn[COLUMNS][col] = col_data
rtn[INDEX] = Binary(lz4.compressHC(np.concatenate(([recs['index'][0].astype('datetime64[ms]').view('uint64')],
np.diff(recs['index'].astype('datetime64[ms]').view('uint64')))
).tostring()))
rtn[INDEX] = Binary(lz4.compressHC(np.concatenate(([recs[index_name][0].astype('datetime64[ms]').view('uint64')],
np.diff(recs[index_name].astype('datetime64[ms]').view('uint64')))).tostring()))
return rtn, final_image

@staticmethod
Expand Down
17 changes: 17 additions & 0 deletions tests/integration/tickstore/test_ts_write.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,20 @@ def test_ts_write_pandas(tickstore_lib):

read = tickstore_lib.read('SYM', columns=None)
assert_frame_equal(read, data, check_names=False)


def test_ts_write_named_col(tickstore_lib):
data = DUMMY_DATA
tickstore_lib.write('SYM', data)

data = tickstore_lib.read('SYM')
assert data.index[0] == dt(2013, 1, 1, tzinfo=mktz('Europe/London'))
assert data.a[0] == 1
assert(data.index.name is None)
data.index.name = 'IndexName'

tickstore_lib.delete('SYM')
tickstore_lib.write('SYM', data)

read = tickstore_lib.read('SYM')
assert(read.index.name is None)

0 comments on commit 43840f7

Please sign in to comment.