Skip to content

datajoint/table.py: smarter dataframe conversion (#666) #776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eywalker merged 5 commits into from
May 15, 2020
Merged

datajoint/table.py: smarter dataframe conversion (#666) #776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
95a705c
datajoint/table.py: smarter dataframe conversion
Apr 30, 2020
172f4bf
datajoint/table.py: update pandas index conversion logic per #776 dis…
May 15, 2020
a3b7c65
tests/test_relation.py: add user-created pd.DataFrame test for #666
May 15, 2020
e88838a
Merge remote-tracking branch 'upstream/master' into issue-666
May 15, 2020
18beaf2
tests/test_relation.py: adjust pd.DataFrame comments to doctsrings
May 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion datajoint/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,11 @@ def insert(self, rows, replace=False, skip_duplicates=False, ignore_extra_fields
"""

if isinstance(rows, pandas.DataFrame):
rows = rows.to_records()
# drop 'extra' synthetic index for 1-field index case -
# frames with more advanced indices should be prepared by user.
rows = rows.reset_index(
Comment thread
ixcat marked this conversation as resolved.
drop=len(rows.index.names) == 1 and not rows.index.names[0]
).to_records(index=False)

# prohibit direct inserts into auto-populated tables
if not allow_direct_insert and not getattr(self, '_allow_insert', True): # allow_insert is only used in AutoPopulate
Expand Down
17 changes: 16 additions & 1 deletion tests/test_relation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ def test_insert_select(self):
'real_id', 'date_of_birth', 'subject_notes', subject_id='subject_id+1000', species='"human"'))
assert_equal(len(self.subject), 2*original_length)

def test_insert_pandas(self):
def test_insert_pandas_roundtrip(self):
''' ensure fetched frames can be inserted '''
schema.TTest2.delete()
n = len(schema.TTest())
assert_true(n > 0)
Expand All @@ -113,6 +114,20 @@ def test_insert_pandas(self):
schema.TTest2.insert(df)
assert_equal(len(schema.TTest2()), n)

def test_insert_pandas_userframe(self):
'''
ensure simple user-created frames (1 field, non-custom index)
can be inserted without extra index adjustment
'''
schema.TTest2.delete()
n = len(schema.TTest())
assert_true(n > 0)
df = pandas.DataFrame(schema.TTest.fetch())
assert_true(isinstance(df, pandas.DataFrame))
assert_equal(len(df), n)
schema.TTest2.insert(df)
assert_equal(len(schema.TTest2()), n)

@raises(dj.DataJointError)
def test_insert_select_ignore_extra_fields0(self):
""" need ignore extra fields for insert select """
Expand Down