From c30fa089b1822f2812b901366dd839ca86812f34 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Wed, 3 Feb 2016 17:42:02 -0600 Subject: [PATCH 1/9] move repr to relational_operand and write repr for fetch object --- datajoint/fetch.py | 23 +++++++++-------------- datajoint/relational_operand.py | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/datajoint/fetch.py b/datajoint/fetch.py index 4474482ac..647e4078b 100644 --- a/datajoint/fetch.py +++ b/datajoint/fetch.py @@ -2,6 +2,8 @@ from collections.abc import Callable, Iterable from functools import wraps import warnings +from itertools import starmap + from .blob import unpack import numpy as np from datajoint import DataJointError @@ -39,6 +41,7 @@ def copy_first(f): """ Decorates methods that return an altered copy of self """ + @wraps(f) def ret(*args, **kwargs): args = list(args) @@ -148,7 +151,7 @@ def __call__(self, **kwargs): if behavior['limit'] is None and behavior['offset'] is not None: warnings.warn('Offset set, but no limit. Setting limit to a large number. ' 'Consider setting a limit explicitly.') - behavior['limit'] = 2*len(self._relation) + behavior['limit'] = 2 * len(self._relation) cur = self._relation.cursor(**behavior) heading = self._relation.heading @@ -217,19 +220,11 @@ def __getitem__(self, item): return return_values[0] if single_output else return_values def __repr__(self): - limit = config['display.limit'] - width = config['display.width'] - rel = self._relation.project(*self._relation.heading.non_blobs) # project out blobs - template = '%%-%d.%ds' % (width, width) - columns = rel.heading.names - repr_string = ' '.join([template % column for column in columns]) + '\n' - repr_string += ' '.join(['+' + '-' * (width - 2) + '+' for _ in columns]) + '\n' - for tup in rel.fetch(limit=limit): - repr_string += ' '.join([template % column for column in tup]) + '\n' - if len(rel) > limit: - repr_string += '...\n' - repr_string += ' (%d tuples)\n' % len(rel) - return repr_string + repr_str = """Fetch object for {items} items on {name}\n""".format(name=self._relation.__class__.__name__, + items=len(self._relation) ) + repr_str += '\n'.join( + ["\t{key}:\t{value}".format(key=k, value=str(v)) for k, v in self.behavior.items() if v is not None]) + return repr_str def __len__(self): return len(self._relation) diff --git a/datajoint/relational_operand.py b/datajoint/relational_operand.py index 698201015..0201d6845 100644 --- a/datajoint/relational_operand.py +++ b/datajoint/relational_operand.py @@ -244,9 +244,25 @@ def _repr_helper(self): """ def __repr__(self): - ret = self._repr_helper() - if self._restrictions: - ret += ' & %r' % self._restrictions + if config['loglevel'].lower() == 'debug': + ret = self._repr_helper() + if self._restrictions: + ret += ' & %r' % self._restrictions + else: + limit = config['display.limit'] + width = config['display.width'] + rel = self.project(*self.heading.non_blobs) # project out blobs + template = '%%-%d.%ds' % (width, width) + columns = rel.heading.names + repr_string = ' '.join([template % column for column in columns]) + '\n' + repr_string += ' '.join(['+' + '-' * (width - 2) + '+' for _ in columns]) + '\n' + for tup in rel.fetch(limit=limit): + repr_string += ' '.join([template % column for column in tup]) + '\n' + if len(rel) > limit: + repr_string += '...\n' + repr_string += ' (%d tuples)\n' % len(rel) + return repr_string + return ret def _repr_html_(self): @@ -259,8 +275,6 @@ def _repr_html_(self): ['\n'.join(['%s' % column for column in tup]) for tup in rel.fetch(limit=limit)]), tuples=len(rel) ) - - return """
\n \n \n From f546c18337e3597f114290b07537956f511b4ad8 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Wed, 3 Feb 2016 17:48:19 -0600 Subject: [PATCH 2/9] cleanup import --- datajoint/fetch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datajoint/fetch.py b/datajoint/fetch.py index 647e4078b..ffbea5c75 100644 --- a/datajoint/fetch.py +++ b/datajoint/fetch.py @@ -2,7 +2,6 @@ from collections.abc import Callable, Iterable from functools import wraps import warnings -from itertools import starmap from .blob import unpack import numpy as np From dc01d7a054904128acebcc4a0ca4608d450af806 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 4 Feb 2016 15:25:44 -0600 Subject: [PATCH 3/9] try to resurrect coveralls --- .coveralls.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 000000000..2eeede133 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1 @@ +repo_token: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY \ No newline at end of file From 737c7d13af01c78c01881562e237613416aaa555 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 4 Feb 2016 18:36:51 -0600 Subject: [PATCH 4/9] second try to resurrect coveralls --- .coveragerc | 6 ++++++ .travis.yml | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..0f46b73ec --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +source = datajoint +omit = + */tests/* + */recycle_bin/* + */1st_version_thunderfish/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 20c2f651b..a19e4979a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,5 +10,7 @@ install: # command to run tests script: - nosetests -vv --with-coverage --cover-package=datajoint + - coverage run --source=datajoint setup.py test + after_success: - coveralls From 12ac78429aff938d167a364c123f135e410a6922 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 4 Feb 2016 19:13:03 -0600 Subject: [PATCH 5/9] third attempt --- .coveragerc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0f46b73ec..d41af3df7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,2 @@ [run] source = datajoint -omit = - */tests/* - */recycle_bin/* - */1st_version_thunderfish/* \ No newline at end of file From 7d0848f0c2d11c202531c1d0e9279bbab3a0332e Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 4 Feb 2016 19:30:54 -0600 Subject: [PATCH 6/9] coveralls 4th --- .coveragerc | 3 +++ .travis.yml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index d41af3df7..8e9b6dd17 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,5 @@ [run] +branch = False source = datajoint + +[report] diff --git a/.travis.yml b/.travis.yml index a19e4979a..c52c11a1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ install: # command to run tests script: - nosetests -vv --with-coverage --cover-package=datajoint - - coverage run --source=datajoint setup.py test after_success: - coveralls From ae6b26cb8935e5d0f8ca379e9fc6a8cd4cc46742 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 9 Feb 2016 12:18:23 -0600 Subject: [PATCH 7/9] test case for issue #190 --- tests/schema.py | 6 +++--- tests/schema_simple.py | 27 +++++++++++++++++++++++++++ tests/test_relational_operand.py | 10 +++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/tests/schema.py b/tests/schema.py index eb9aa8cea..494e4066a 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -3,6 +3,7 @@ """ import random + import numpy as np import datajoint as dj from . import PREFIX, CONN_INFO @@ -55,7 +56,6 @@ def _prepare(self): @schema class Language(dj.Lookup): - definition = """ # languages spoken by some of the developers @@ -99,7 +99,7 @@ def _make_tuples(self, key): self.insert1( dict(key, experiment_id=experiment_id, - experiment_date=(date.today()-timedelta(random.expovariate(1/30))).isoformat(), + experiment_date=(date.today() - timedelta(random.expovariate(1 / 30))).isoformat(), username=random.choice(users))) @@ -121,7 +121,7 @@ def _make_tuples(self, key): self.insert1( dict(key, trial_id=trial_id, - start_time=random.random()*1e9 + start_time=random.random() * 1e9 )) diff --git a/tests/schema_simple.py b/tests/schema_simple.py index 61fedbb66..e5478cfe9 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -5,6 +5,7 @@ import datajoint as dj from . import PREFIX, CONN_INFO + schema = dj.schema(PREFIX + '_relational', locals(), connection=dj.conn(**CONN_INFO)) @@ -102,3 +103,29 @@ def _make_tuples(self, key): for i, ref in enumerate(references): if random.getrandbits(1): sub.insert1(dict(key, id_f=i, **ref)) + + +@schema +class DataA(dj.Lookup): + definition = """ + idx : int + --- + a : int + """ + + @property + def contents(self): + yield from zip(range(5), range(5)) + + +@schema +class DataB(dj.Lookup): + definition = """ + idx : int + --- + a : int + """ + + @property + def contents(self): + yield from zip(range(5), range(5, 10)) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index 2eee7be7d..72cd2a275 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -3,7 +3,7 @@ assert_false, assert_true, assert_list_equal, \ assert_tuple_equal, assert_dict_equal, raises import datajoint as dj -from .schema_simple import A, B, D, E, L +from .schema_simple import A, B, D, E, L, DataA, DataB import datetime from .schema import Experiment @@ -182,3 +182,11 @@ def test_datetime(): e1 = Experiment() & dict(experiment_date=str(date)) e2 = Experiment() & dict(experiment_date=date) assert_true(len(e1) == len(e2) > 0, 'Two date restriction do not yield the same result') + + @staticmethod + def test_join_project_optimization(): + """Test optimization for join of projected relations with matching non-primary key""" + print(DataA().project() * DataB().project()) + print(DataA()) + assert_true(len(DataA().project() * DataB().project()) == len(DataA()) == len(DataB()), + "Join of projected relations does not work") From 709e45738440c8edd1341cce7b868f0804e6326d Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Fri, 12 Feb 2016 19:19:41 -0600 Subject: [PATCH 8/9] fix AND-List catastrophe --- datajoint/base_relation.py | 2 +- datajoint/relational_operand.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 1775c783d..bbe2be3f5 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -288,7 +288,7 @@ def delete(self): restrictions = defaultdict(list) if self.restrictions: restrict_by_me.add(self.full_table_name) - restrictions[self.full_table_name] = self.restrictions # copy own restrictions + restrictions[self.full_table_name].append(self.restrictions) # copy own restrictions for r in relations.values(): restrict_by_me.update(r.references) for name, r in relations.items(): diff --git a/datajoint/relational_operand.py b/datajoint/relational_operand.py index 0201d6845..0187d1c32 100644 --- a/datajoint/relational_operand.py +++ b/datajoint/relational_operand.py @@ -45,24 +45,26 @@ def where_clause(self): """ def make_condition(arg, _negate=False): - if isinstance(arg, (str, AndList)): - return str(arg), _negate + if isinstance(arg, str): + return arg, _negate + elif isinstance(arg, AndList): + return '(' + ' AND '.join([make_condition(element)[0] for element in arg]) + ')', _negate # semijoin or antijoin - if isinstance(arg, RelationalOperand): + elif isinstance(arg, RelationalOperand): common_attributes = [q for q in self.heading.names if q in arg.heading.names] if not common_attributes: - condition = 'FALSE' if negate else 'TRUE' + condition = 'FALSE' if _negate else 'TRUE' else: common_attributes = '`' + '`,`'.join(common_attributes) + '`' condition = '({fields}) {not_}in ({subquery})'.format( fields=common_attributes, - not_="not " if negate else "", + not_="not " if _negate else "", subquery=arg.make_select(common_attributes)) - return condition, False # negate is cleared + return condition, False # _negate is cleared # mappings are turned into ANDed equality conditions - if isinstance(arg, Mapping): + elif isinstance(arg, Mapping): condition = ['`%s`=%r' % (k, v if not isinstance(v, (datetime.date, datetime.datetime, datetime.time)) else str(v)) for k, v in arg.items() if k in self.heading] @@ -91,6 +93,8 @@ def make_condition(arg, _negate=False): conditions.append(('NOT (%s)' if negate else '(%s)') % item) return ' WHERE ' + ' AND '.join(conditions) + def __repr__(self): + return 'AND List: ' + repr(self._list) class RelationalOperand(metaclass=abc.ABCMeta): """ From f666a84faa7ed86fca5f0b0aaa85491de2d635cf Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Mon, 15 Feb 2016 10:27:34 -0600 Subject: [PATCH 9/9] set default loglevel to INFO --- datajoint/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/settings.py b/datajoint/settings.py index 8e239dd2b..ff43c490d 100644 --- a/datajoint/settings.py +++ b/datajoint/settings.py @@ -35,7 +35,7 @@ # 'connection.init_function': None, # - 'loglevel': 'DEBUG', + 'loglevel': 'INFO', # 'safemode': True, #