From 8fe96deb98ad194fe60eee0a02644e69cc5974e5 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 21 Aug 2015 20:05:12 -0500 Subject: [PATCH 1/8] implementing suborinate relations as nested classes --- datajoint/schema.py | 4 +- datajoint/user_relations.py | 75 +++++++++--------- tests/schema_simple.py | 109 ++++++++++++++++++++++++++ tests/test_cascading_delete.py | 74 ++++++++++++++++++ tests/test_relational_operand.py | 129 +++---------------------------- 5 files changed, 235 insertions(+), 156 deletions(-) create mode 100644 tests/schema_simple.py create mode 100644 tests/test_cascading_delete.py diff --git a/datajoint/schema.py b/datajoint/schema.py index 2f538a2f0..f45f2f74b 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -4,7 +4,8 @@ from . import conn from . import DataJointError from .heading import Heading - +from .user_relations import Sub +from .autopopulate import AutoPopulate logger = logging.getLogger(__name__) @@ -54,6 +55,7 @@ def __call__(self, cls): instance = cls() instance.heading instance._prepare() + return cls @property diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 636e86b10..b14ae341f 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -5,9 +5,37 @@ from datajoint.relation import Relation from .autopopulate import AutoPopulate from .utils import from_camel_case +from . import DataJointError -class Manual(Relation): +class Sub(Relation): + + @property + def master(self): + if not hasattr(self, '_master'): + raise DataJointError( + 'subordinate relations must be declared inside a base relation class') + return self._master + + @property + def table_name(self): + return self.master.table_name + '__' + from_camel_case(self.__class__.__name__) + + +class MasterMeta(type): + """ + The metaclass for master relations. Assigns the class into the _master property of all + properties of type Sub. + """ + def __new__(cls, name, parents, dct): + for value in dct.values(): + if issubclass(value, Sub): + value._master = cls + return super().__new__(cls, name, parents, dct) + + + +class Manual(Relation, metaclass=MasterMeta): """ Inherit from this class if the table's values are entered manually. """ @@ -20,7 +48,7 @@ def table_name(self): return from_camel_case(self.__class__.__name__) -class Lookup(Relation): +class Lookup(ClassedRelation): """ Inherit from this class if the table's values are for lookup. This is currently equivalent to defining the table as Manual and serves semantic @@ -42,7 +70,7 @@ def _prepare(self): self.insert(self.contents, ignore_errors=True) -class Imported(Relation, AutoPopulate): +class Imported(ClassedRelation, AutoPopulate): """ Inherit from this class if the table's values are imported from external data sources. The inherited class must at least provide the function `_make_tuples`. @@ -56,7 +84,7 @@ def table_name(self): return "_" + from_camel_case(self.__class__.__name__) -class Computed(Relation, AutoPopulate): +class Computed(ClassedRelation, AutoPopulate): """ Inherit from this class if the table's values are computed from other relations in the schema. The inherited class must at least provide the function `_make_tuples`. @@ -70,42 +98,19 @@ def table_name(self): return "__" + from_camel_case(self.__class__.__name__) -class Subordinate: +class Subordinate(dj.Relation): """ - Mix-in to make computed tables subordinate - """ - - @property - def populated_from(self): - """ - Overrides the `populate_from` property because subtables should not be populated - directly. - - :return: None - """ - return None + Subordinate relation is declared within another relation class - def _make_tuples(self, key): - """ - Overrides the `_make_tuples` property because subtables should not be populated - directly. Raises an error if this method is called (usually from populate of the - inheriting object). + :input master: instance of the master relation containing the suborinate relation + """ - :raises: NotImplementedError - """ - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') + def __init__(self, master): + self._master = master - def progress(self): - """ - Overrides the `progress` method because subtables should not be populated directly. - """ - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') + def table_name(self): + return self._master.table_name + '__' + from_camel_case(self.__class__.__name__) - def populate(self, *args, **kwargs): - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') diff --git a/tests/schema_simple.py b/tests/schema_simple.py new file mode 100644 index 000000000..b8ae90e2a --- /dev/null +++ b/tests/schema_simple.py @@ -0,0 +1,109 @@ +""" +A simple, abstract schema to test relational algebra +""" +import random +import datajoint as dj +from . import PREFIX, CONN_INFO + +schema = dj.schema(PREFIX + '_relational', locals(), connection=dj.conn(**CONN_INFO)) + + +@schema +class A(dj.Lookup): + definition = """ + id_a :int + --- + cond_in_a :tinyint + """ + contents = [(i, i % 4 > i % 3) for i in range(10)] + + +@schema +class B(dj.Computed): + definition = """ + -> A + id_b :int + --- + mu :float # mean value + sigma :float # standard deviation + n :smallint # number samples + """ + + def _make_tuples(self, key): + random.seed(str(key)) + sub = C() + for i in range(4): + key['id_b'] = i + mu = random.normalvariate(0, 10) + sigma = random.lognormvariate(0, 4) + n = random.randint(0, 10) + self.insert1(dict(key, mu=mu, sigma=sigma, n=n)) + for j in range(n): + sub.insert1(dict(key, id_c=j, value=random.normalvariate(mu, sigma))) + + +@schema +class C(dj.Subordinate, dj.Computed): + definition = """ + -> B + id_c :int + --- + value :float # normally distributed variables according to parameters in B + """ + + +@schema +class L(dj.Lookup): + definition = """ + id_l: int + --- + cond_in_l :tinyint + """ + contents = ((i, i % 3 >= i % 5) for i in range(30)) + + +@schema +class D(dj.Computed): + definition = """ + -> A + id_d :int + --- + -> L + """ + + def _make_tuples(self, key): + # connect to random L + random.seed(str(key)) + lookup = list(L().fetch.keys()) + for i in range(4): + self.insert1(dict(key, id_d=i, **random.choice(lookup))) + + +@schema +class E(dj.Computed): + definition = """ + -> B + -> D + --- + -> L + """ + + def _make_tuples(self, key): + random.seed(str(key)) + self.insert1(dict(key, **random.choice(list(L().fetch.keys())))) + sub = F() + references = list((C() & key).fetch.keys()) + random.shuffle(references) + for i, ref in enumerate(references): + if random.getrandbits(1): + sub.insert1(dict(key, id_f=i, **ref)) + + +@schema +class F(dj.Subordinate, dj.Computed): + definition = """ + -> E + id_f :int + --- + -> C + """ \ No newline at end of file diff --git a/tests/test_cascading_delete.py b/tests/test_cascading_delete.py new file mode 100644 index 000000000..1af7dc50a --- /dev/null +++ b/tests/test_cascading_delete.py @@ -0,0 +1,74 @@ +from nose.tools import assert_false, assert_true +import datajoint as dj +from .schema_simple import A, B, C, D, E, F, L + + +class TestDelete: + + @staticmethod + def setup(): + """ + class-level test setup. Executes before each test method. + """ + A()._prepare() + L()._prepare() + B().populate() + D().populate() + E().populate() + + @staticmethod + def test_delete_tree(): + assert_false(dj.config['safemode'], 'safemode must be off for testing') + assert_true(L() and A() and B() and C() and D() and E() and F(), + 'schema is not populated') + A().delete() + assert_false(A() or B() or C() or D() or E() or F(), 'incomplete delete') + + @staticmethod + def test_delete_tree_restricted(): + assert_false(dj.config['safemode'], 'safemode must be off for testing') + assert_true(L() and A() and B() and C() and D() and E() and F(), + 'schema is not populated') + cond = 'cond_in_a' + rel = A() & cond + rest = dict( + A=len(A())-len(cond), + B=len(B()-rel), + C=len(C()-rel), + D=len(D()-rel), + E=len(E()-rel), + F=len(F()-rel) + ) + rel.delete() + assert_false(rel or + (B() & rel) or + (C() & rel) or + (D() & rel) or + (E() & rel) or + (F() & rel), + 'incomplete delete') + assert_true( + len(A()) == rest['A'] and + len(B()) == rest['B'] and + len(C()) == rest['C'] and + len(D()) == rest['D'] and + len(E()) == rest['E'] and + len(F()) == rest['F'], + 'incorrect restricted delete') + + @staticmethod + def test_delete_lookup(): + assert_false(dj.config['safemode'], 'safemode must be off for testing') + assert_true(L() and A() and B() and C() and D() and E() and F(), + 'schema is not populated') + L().delete() + assert_false(bool(L() or D() or E() or F()), 'incomplete delete') + A().delete() # delete all is necessary because delete L deletes from subtables. TODO: submit this as an issue + + # @staticmethod + # def test_delete_lookup_restricted(): + # assert_false(dj.config['safemode'], 'safemode must be off for testing') + # assert_true(L() and A() and B() and C() and D() and E() and F(), + # 'schema is not populated') + # rel = L() & 'cond_in_l' + # L().delete() diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index a6c19a46e..a9bca66db 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -1,114 +1,9 @@ -import datajoint as dj -from . import PREFIX, CONN_INFO -import random import numpy as np from nose.tools import assert_raises, assert_equal, \ assert_false, assert_true, assert_list_equal, \ assert_tuple_equal, assert_dict_equal, raises - - -schema = dj.schema(PREFIX + '_relational', locals(), connection=dj.conn(**CONN_INFO)) - - -@schema -class A(dj.Lookup): - definition = """ - id_a :int - --- - cond_in_a :tinyint - """ - contents = [(i, i % 4 > i % 3) for i in range(10)] - - -@schema -class B(dj.Computed): - definition = """ - -> A - id_b :int - --- - mu :float # mean value - sigma :float # standard deviation - n :smallint # number samples - """ - - def _make_tuples(self, key): - random.seed(str(key)) - sub = C() - for i in range(4): - key['id_b'] = i - mu = random.normalvariate(0, 10) - sigma = random.lognormvariate(0, 4) - n = random.randint(0, 10) - self.insert1(dict(key, mu=mu, sigma=sigma, n=n)) - for j in range(n): - sub.insert1(dict(key, id_c=j, value=random.normalvariate(mu, sigma))) - - -@schema -class C(dj.Subordinate, dj.Computed): - definition = """ - -> B - id_c :int - --- - value :float # normally distributed variables according to parameters in B - """ - - -@schema -class L(dj.Lookup): - definition = """ - id_l: int - --- - cond_in_l :tinyint - """ - contents = ((i, i % 3 >= i % 5) for i in range(30)) - - -@schema -class D(dj.Computed): - definition = """ - -> A - id_d :int - --- - -> L - """ - - def _make_tuples(self, key): - # connect to random L - random.seed(str(key)) - lookup = list(L().fetch.keys()) - for i in range(4): - self.insert1(dict(key, id_d=i, **random.choice(lookup))) - - -@schema -class E(dj.Computed): - definition = """ - -> B - -> D - --- - -> L - """ - - def _make_tuples(self, key): - random.seed(str(key)) - self.insert1(dict(key, **random.choice(list(L().fetch.keys())))) - sub = F() - references = list((C() & key).fetch.keys()) - random.shuffle(references) - for i, ref in enumerate(references): - if random.getrandbits(1): - sub.insert1(dict(key, id_f=i, **ref)) - - -@schema -class F(dj.Subordinate, dj.Computed): - definition = """ - -> E - id_f :int - --- - -> C - """ +import datajoint as dj +from .schema_simple import A, B, C, D, E, F, L def setup(): @@ -118,7 +13,6 @@ def setup(): B().populate() D().populate() E().populate() - pass class TestRelational: @@ -217,22 +111,17 @@ def test_project(): 'extend does not work') # projection after restriction - assert_equal( - len(D() & (L() & 'cond_in_l')) + len(D() - (L() & 'cond_in_l')), - len(D()), - 'failed semijoin or antijoin' - ) - assert_equal( - len((D() - (L() & 'cond_in_l')).project()), - len(D() - (L() & 'cond_in_l')), - 'projection altered the cardinality of a restricted relation' - ) + cond = L() & 'cond_in_l' + assert_equal(len(D() & cond) + len(D() - cond), len(D()), + 'failed semijoin or antijoin') + assert_equal(len((D() & cond).project()), len((D() & cond)), + 'projection failed: altered its argument''s cardinality') @staticmethod def test_aggregate(): x = B().aggregate(C(), 'n', count='count(id_c)', mean='avg(value)', max='max(value)') assert_equal(len(x), len(B())) - for n, count, mean, max, key in zip(*x.fetch['n', 'count', 'mean', 'max', dj.key]): + for n, count, mean, max_, key in zip(*x.fetch['n', 'count', 'mean', 'max', dj.key]): assert_equal(n, count, 'aggregation failed (count)') values = (C() & key).fetch['value'] assert_true(bool(len(values)) == bool(n), @@ -240,5 +129,5 @@ def test_aggregate(): if n: assert_true(np.isclose(mean, values.mean(), rtol=1e-4, atol=1e-5), "aggregation failed (mean)") - assert_true(np.isclose(max, values.max(), rtol=1e-4, atol=1e-5), + assert_true(np.isclose(max_, values.max(), rtol=1e-4, atol=1e-5), "aggregation failed (max)") From 574b8a185c32f164c2e935698ae77fc9e494ae45 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 21 Aug 2015 20:10:32 -0500 Subject: [PATCH 2/8] bugfix in declaring subordinate relations --- datajoint/user_relations.py | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index b14ae341f..22b0d2c02 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -34,7 +34,6 @@ def __new__(cls, name, parents, dct): return super().__new__(cls, name, parents, dct) - class Manual(Relation, metaclass=MasterMeta): """ Inherit from this class if the table's values are entered manually. @@ -48,7 +47,7 @@ def table_name(self): return from_camel_case(self.__class__.__name__) -class Lookup(ClassedRelation): +class Lookup(Relation): """ Inherit from this class if the table's values are for lookup. This is currently equivalent to defining the table as Manual and serves semantic @@ -70,7 +69,7 @@ def _prepare(self): self.insert(self.contents, ignore_errors=True) -class Imported(ClassedRelation, AutoPopulate): +class Imported(Relation, AutoPopulate, metaclass=MasterMeta): """ Inherit from this class if the table's values are imported from external data sources. The inherited class must at least provide the function `_make_tuples`. @@ -84,7 +83,7 @@ def table_name(self): return "_" + from_camel_case(self.__class__.__name__) -class Computed(ClassedRelation, AutoPopulate): +class Computed(Relation, AutoPopulate, metaclass=MasterMeta): """ Inherit from this class if the table's values are computed from other relations in the schema. The inherited class must at least provide the function `_make_tuples`. @@ -96,23 +95,3 @@ def table_name(self): :returns: the table name of the table formatted for mysql. """ return "__" + from_camel_case(self.__class__.__name__) - - -class Subordinate(dj.Relation): - """ - Subordinate relation is declared within another relation class - - :input master: instance of the master relation containing the suborinate relation - """ - - def __init__(self, master): - self._master = master - - def table_name(self): - return self._master.table_name + '__' + from_camel_case(self.__class__.__name__) - - - - - - From cbc9d36c269352e367a184140cd0f1b81eb51806 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sat, 22 Aug 2015 15:47:11 -0500 Subject: [PATCH 3/8] restored dj.Subordinate for backward compatibility. Subordinate relations now subclass from dj.Sub --- datajoint/schema.py | 36 +++++++++++++++++++++++++++--------- datajoint/user_relations.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/datajoint/schema.py b/datajoint/schema.py index f45f2f74b..ff4718c98 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -45,19 +45,37 @@ def __call__(self, cls): The decorator binds its argument class object to a database :param cls: class to be decorated """ - # class-level attributes - cls.database = self.database - cls._connection = self.connection - cls._heading = Heading() - cls._context = self.context - # trigger table declaration by requesting the heading from an instance - instance = cls() - instance.heading - instance._prepare() + def process_class(cls): + # class-level attributes + cls.database = self.database + cls._connection = self.connection + cls._heading = Heading() + cls._context = self.context + + # trigger table declaration by requesting the heading from an instance + instance = cls() + instance.heading + instance._prepare() + + if issubclass(cls, Sub): + raise DataJointError( + 'Subordinate relations need not be assigned to a schema directly') + + process_class(cls) + + # assign _master in all subtables and declare them too + for sub in cls.__dict__.values(): + if issubclass(sub, Sub): + sub._master = self + process_class(sub) return cls @property def jobs(self): + """ + schema.jobs provides a view of the job reservation table for the schema + :return: + """ return self.connection.jobs[self.database] diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 22b0d2c02..ccae76160 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -95,3 +95,39 @@ def table_name(self): :returns: the table name of the table formatted for mysql. """ return "__" + from_camel_case(self.__class__.__name__) + + +class Subordinate: + """ + Mix-in to make computed tables subordinate + """ + + @property + def populated_from(self): + """ + Overrides the `populate_from` property because subtables should not be populated + directly. + :return: None + """ + return None + + def _make_tuples(self, key): + """ + Overrides the `_make_tuples` property because subtables should not be populated + directly. Raises an error if this method is called (usually from populate of the + inheriting object). + :raises: NotImplementedError + """ + raise NotImplementedError( + 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') + + def progress(self): + """ + Overrides the `progress` method because subtables should not be populated directly. + """ + raise NotImplementedError( + 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') + + def populate(self, *args, **kwargs): + raise NotImplementedError( + 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') \ No newline at end of file From 0b9744305e1956bf81e0f3916ccc6c1088fee559 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sat, 22 Aug 2015 19:02:40 -0500 Subject: [PATCH 4/8] intermediate commit, work in progress --- datajoint/__init__.py | 9 +++++---- datajoint/schema.py | 24 ++++++++++++---------- datajoint/user_relations.py | 21 +++++-------------- tests/schema.py | 35 +++++++++++++------------------- tests/schema_simple.py | 40 ++++++++++++++++++------------------- tests/test_autopopulate.py | 2 +- tests/test_declare.py | 2 +- 7 files changed, 59 insertions(+), 74 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index b745ab686..5b2776a69 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -17,12 +17,13 @@ 'config', 'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'Relation', 'schema', - 'Manual', 'Lookup', 'Imported', 'Computed', + 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub', 'conn', 'kill'] -# define an object that identifies the primary key in RelationalOperand.__getitem__ -class PrimaryKey: pass +# define an object that identifies the primary key in RelationalOperand.__getitem__ +class PrimaryKey: + pass key = PrimaryKey @@ -54,7 +55,7 @@ class DataJointError(Exception): # ------------- flatten import hierarchy ------------------------- from .connection import conn, Connection from .relation import Relation -from .user_relations import Manual, Lookup, Imported, Computed, Subordinate +from .user_relations import Manual, Lookup, Imported, Computed, Sub, Subordinate from .relational_operand import Not from .heading import Heading from .schema import schema diff --git a/datajoint/schema.py b/datajoint/schema.py index ff4718c98..fd1c28e9e 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -1,11 +1,10 @@ import pymysql import logging -from . import conn -from . import DataJointError +from . import conn, DataJointError from .heading import Heading +from .relation import Relation from .user_relations import Sub -from .autopopulate import AutoPopulate logger = logging.getLogger(__name__) @@ -47,7 +46,9 @@ def __call__(self, cls): """ def process_class(cls): - # class-level attributes + """ + assign schema properties to the relation class + """ cls.database = self.database cls._connection = self.connection cls._heading = Heading() @@ -64,11 +65,14 @@ def process_class(cls): process_class(cls) - # assign _master in all subtables and declare them too - for sub in cls.__dict__.values(): - if issubclass(sub, Sub): - sub._master = self - process_class(sub) + # assign _master in all subordinates; declare subordinate relations + for sub in (cls.__getattribute__(sub) for sub in dir(cls)): + if type(sub) is type: + if issubclass(sub, Sub): + sub._master = self + process_class(sub) + elif issubclass(sub, Relation): + raise DataJointError('Subordinate relations must subclass from datajoint.Sub') return cls @@ -76,6 +80,6 @@ def process_class(cls): def jobs(self): """ schema.jobs provides a view of the job reservation table for the schema - :return: + :return: jobs relation """ return self.connection.jobs[self.database] diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index ccae76160..2d2235696 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -22,19 +22,7 @@ def table_name(self): return self.master.table_name + '__' + from_camel_case(self.__class__.__name__) -class MasterMeta(type): - """ - The metaclass for master relations. Assigns the class into the _master property of all - properties of type Sub. - """ - def __new__(cls, name, parents, dct): - for value in dct.values(): - if issubclass(value, Sub): - value._master = cls - return super().__new__(cls, name, parents, dct) - - -class Manual(Relation, metaclass=MasterMeta): +class Manual(Relation): """ Inherit from this class if the table's values are entered manually. """ @@ -69,7 +57,7 @@ def _prepare(self): self.insert(self.contents, ignore_errors=True) -class Imported(Relation, AutoPopulate, metaclass=MasterMeta): +class Imported(Relation, AutoPopulate): """ Inherit from this class if the table's values are imported from external data sources. The inherited class must at least provide the function `_make_tuples`. @@ -83,7 +71,7 @@ def table_name(self): return "_" + from_camel_case(self.__class__.__name__) -class Computed(Relation, AutoPopulate, metaclass=MasterMeta): +class Computed(Relation, AutoPopulate): """ Inherit from this class if the table's values are computed from other relations in the schema. The inherited class must at least provide the function `_make_tuples`. @@ -99,7 +87,8 @@ def table_name(self): class Subordinate: """ - Mix-in to make computed tables subordinate + Mix-in to make computed tables subordinate. + This class is DEPRECATED and will be removed in a future version. Use dj.Sub instead. """ @property diff --git a/tests/schema.py b/tests/schema.py index 5ae94480f..2bc9d9dde 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -136,36 +136,29 @@ class Ephys(dj.Imported): duration :double # (s) """ + class Channel(dj.Sub): + definition = """ # subtable containing individual channels + -> Ephys + channel :tinyint unsigned # channel number within Ephys + ---- + voltage :longblob + """ + def _make_tuples(self, key): """ populate with random data """ - random.seed('Amazing seed') + random.seed(str(key)) row = dict(key, sampling_frequency=6000, duration=np.minimum(2, random.expovariate(1))) self.insert1(row) number_samples = round(row['duration'] * row['sampling_frequency']) - EphysChannel().fill(key, number_samples=number_samples) - - -@schema -class EphysChannel(dj.Subordinate, dj.Imported): - definition = """ # subtable containing individual channels - -> Ephys - channel :tinyint unsigned # channel number within Ephys - ---- - voltage :longblob - """ - - def fill(self, key, number_samples): - """ - populate random trace of specified length - """ - random.seed('Amazing seed') + sub = Ephys.Channel() for channel in range(2): - self.insert1( + sub.insert1( dict(key, channel=channel, - voltage=np.float32(np.random.randn(number_samples)) - )) + voltage=np.float32(np.random.randn(number_samples)))) + + diff --git a/tests/schema_simple.py b/tests/schema_simple.py index b8ae90e2a..a7e7239a8 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -29,9 +29,17 @@ class B(dj.Computed): n :smallint # number samples """ + class C(dj.Sub): + definition = """ + -> B + id_c :int + --- + value :float # normally distributed variables according to parameters in B + """ + def _make_tuples(self, key): random.seed(str(key)) - sub = C() + sub = B.C() for i in range(4): key['id_b'] = i mu = random.normalvariate(0, 10) @@ -42,16 +50,6 @@ def _make_tuples(self, key): sub.insert1(dict(key, id_c=j, value=random.normalvariate(mu, sigma))) -@schema -class C(dj.Subordinate, dj.Computed): - definition = """ - -> B - id_c :int - --- - value :float # normally distributed variables according to parameters in B - """ - - @schema class L(dj.Lookup): definition = """ @@ -88,22 +86,22 @@ class E(dj.Computed): -> L """ + class F(dj.Sub): + definition = """ + -> E + id_f :int + --- + -> B.C + """ + def _make_tuples(self, key): random.seed(str(key)) self.insert1(dict(key, **random.choice(list(L().fetch.keys())))) - sub = F() - references = list((C() & key).fetch.keys()) + sub = E.F() + references = list((B.C() & key).fetch.keys()) random.shuffle(references) for i, ref in enumerate(references): if random.getrandbits(1): sub.insert1(dict(key, id_f=i, **ref)) -@schema -class F(dj.Subordinate, dj.Computed): - definition = """ - -> E - id_f :int - --- - -> C - """ \ No newline at end of file diff --git a/tests/test_autopopulate.py b/tests/test_autopopulate.py index cbb5142a4..31bc6496f 100644 --- a/tests/test_autopopulate.py +++ b/tests/test_autopopulate.py @@ -16,7 +16,7 @@ def __init__(self): self.experiment = schema.Experiment() self.trial = schema.Trial() self.ephys = schema.Ephys() - self.channel = schema.EphysChannel() + self.channel = schema.Ephys.Channel() # delete automatic tables just in case self.channel.delete_quick() diff --git a/tests/test_declare.py b/tests/test_declare.py index d856c750c..b00545599 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -8,7 +8,7 @@ experiment = schema.Experiment() trial = schema.Trial() ephys = schema.Ephys() -channel = schema.EphysChannel() +channel = schema.Ephys.Channel() class TestDeclare: From 18aa7d431ca71aadaa92a69b8b91321b3d403605 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sun, 23 Aug 2015 18:59:37 -0500 Subject: [PATCH 5/8] fixed issue #146: Subordinate relations are implemented as nested classes. --- datajoint/schema.py | 38 +++++++++++++++++--------------- datajoint/user_relations.py | 2 +- tests/schema_simple.py | 3 +-- tests/test_cascading_delete.py | 26 +++++++++++----------- tests/test_relation.py | 2 +- tests/test_relational_operand.py | 12 +++++----- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/datajoint/schema.py b/datajoint/schema.py index fd1c28e9e..c6739d837 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -45,35 +45,37 @@ def __call__(self, cls): :param cls: class to be decorated """ - def process_class(cls): + def process_relation_class(class_object, context): """ - assign schema properties to the relation class + assign schema properties to the relation class and declare the table """ - cls.database = self.database - cls._connection = self.connection - cls._heading = Heading() - cls._context = self.context - - # trigger table declaration by requesting the heading from an instance - instance = cls() - instance.heading + class_object.database = self.database + class_object._connection = self.connection + class_object._heading = Heading() + class_object._context = context + instance = class_object() + instance.heading # trigger table declaration instance._prepare() if issubclass(cls, Sub): raise DataJointError( 'Subordinate relations need not be assigned to a schema directly') - process_class(cls) + process_relation_class(cls, context=self.context) - # assign _master in all subordinates; declare subordinate relations - for sub in (cls.__getattribute__(sub) for sub in dir(cls)): - if type(sub) is type: - if issubclass(sub, Sub): - sub._master = self - process_class(sub) + # Process subordinate relations + for name in (name for name in dir(cls) if not name.startswith('_')): + sub = getattr(cls, name) + try: + is_sub = issubclass(sub, Sub) + except TypeError: + pass + else: + if is_sub: + sub._master = cls + process_relation_class(sub, context=dict(self.context, **{cls.__name__: cls})) elif issubclass(sub, Relation): raise DataJointError('Subordinate relations must subclass from datajoint.Sub') - return cls @property diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 2d2235696..65eb354c7 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -19,7 +19,7 @@ def master(self): @property def table_name(self): - return self.master.table_name + '__' + from_camel_case(self.__class__.__name__) + return self.master().table_name + '__' + from_camel_case(self.__class__.__name__) class Manual(Relation): diff --git a/tests/schema_simple.py b/tests/schema_simple.py index a7e7239a8..cf7eb4289 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -57,7 +57,7 @@ class L(dj.Lookup): --- cond_in_l :tinyint """ - contents = ((i, i % 3 >= i % 5) for i in range(30)) + contents = [(i, i % 3 >= i % 5) for i in range(30)] @schema @@ -104,4 +104,3 @@ def _make_tuples(self, key): if random.getrandbits(1): sub.insert1(dict(key, id_f=i, **ref)) - diff --git a/tests/test_cascading_delete.py b/tests/test_cascading_delete.py index 1af7dc50a..8182ad6c1 100644 --- a/tests/test_cascading_delete.py +++ b/tests/test_cascading_delete.py @@ -1,6 +1,6 @@ from nose.tools import assert_false, assert_true import datajoint as dj -from .schema_simple import A, B, C, D, E, F, L +from .schema_simple import A, B, D, E, L class TestDelete: @@ -19,50 +19,50 @@ def setup(): @staticmethod def test_delete_tree(): assert_false(dj.config['safemode'], 'safemode must be off for testing') - assert_true(L() and A() and B() and C() and D() and E() and F(), + assert_true(L() and A() and B() and B.C() and D() and E() and E.F(), 'schema is not populated') A().delete() - assert_false(A() or B() or C() or D() or E() or F(), 'incomplete delete') + assert_false(A() or B() or B.C() or D() or E() or E.F(), 'incomplete delete') @staticmethod def test_delete_tree_restricted(): assert_false(dj.config['safemode'], 'safemode must be off for testing') - assert_true(L() and A() and B() and C() and D() and E() and F(), + assert_true(L() and A() and B() and B.C() and D() and E() and E.F(), 'schema is not populated') cond = 'cond_in_a' rel = A() & cond rest = dict( - A=len(A())-len(cond), + A=len(A())-len(rel), B=len(B()-rel), - C=len(C()-rel), + C=len(B.C()-rel), D=len(D()-rel), E=len(E()-rel), - F=len(F()-rel) + F=len(E.F()-rel) ) rel.delete() assert_false(rel or (B() & rel) or - (C() & rel) or + (B.C() & rel) or (D() & rel) or (E() & rel) or - (F() & rel), + (E.F() & rel), 'incomplete delete') assert_true( len(A()) == rest['A'] and len(B()) == rest['B'] and - len(C()) == rest['C'] and + len(B.C()) == rest['C'] and len(D()) == rest['D'] and len(E()) == rest['E'] and - len(F()) == rest['F'], + len(E.F()) == rest['F'], 'incorrect restricted delete') @staticmethod def test_delete_lookup(): assert_false(dj.config['safemode'], 'safemode must be off for testing') - assert_true(L() and A() and B() and C() and D() and E() and F(), + assert_true(bool(L() and A() and B() and B.C() and D() and E() and E.F()), 'schema is not populated') L().delete() - assert_false(bool(L() or D() or E() or F()), 'incomplete delete') + assert_false(bool(L() or D() or E() or E.F()), 'incomplete delete') A().delete() # delete all is necessary because delete L deletes from subtables. TODO: submit this as an issue # @staticmethod diff --git a/tests/test_relation.py b/tests/test_relation.py index d025775d0..a2de3749b 100644 --- a/tests/test_relation.py +++ b/tests/test_relation.py @@ -18,7 +18,7 @@ def __init__(self): self.experiment = schema.Experiment() self.trial = schema.Trial() self.ephys = schema.Ephys() - self.channel = schema.EphysChannel() + self.channel = schema.Ephys.Channel() def test_contents(self): """ diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index a9bca66db..5e0e68b67 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -3,13 +3,15 @@ assert_false, assert_true, assert_list_equal, \ assert_tuple_equal, assert_dict_equal, raises import datajoint as dj -from .schema_simple import A, B, C, D, E, F, L +from .schema_simple import A, B, D, E, L def setup(): """ module-level test setup """ + A()._prepare() + L()._prepare() B().populate() D().populate() E().populate() @@ -24,10 +26,10 @@ def test_populate(): assert_false(E().progress(display=False)[0], 'E incompletely populated') assert_true(len(B()) == 40, 'B populated incorrectly') - assert_true(len(C()) > 0, 'C populated incorrectly') + assert_true(len(B.C()) > 0, 'C populated incorrectly') assert_true(len(D()) == 40, 'D populated incorrectly') assert_true(len(E()) == len(B())*len(D())/len(A()), 'E populated incorrectly') - assert_true(len(F()) > 0, 'F populated incorrectly') + assert_true(len(E.F()) > 0, 'F populated incorrectly') @staticmethod def test_join(): @@ -119,11 +121,11 @@ def test_project(): @staticmethod def test_aggregate(): - x = B().aggregate(C(), 'n', count='count(id_c)', mean='avg(value)', max='max(value)') + x = B().aggregate(B.C(), 'n', count='count(id_c)', mean='avg(value)', max='max(value)') assert_equal(len(x), len(B())) for n, count, mean, max_, key in zip(*x.fetch['n', 'count', 'mean', 'max', dj.key]): assert_equal(n, count, 'aggregation failed (count)') - values = (C() & key).fetch['value'] + values = (B.C() & key).fetch['value'] assert_true(bool(len(values)) == bool(n), 'aggregation failed (restriction)') if n: From 7a6600bd1e6fa76e6f906f97f5e2b914abf497e6 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sun, 23 Aug 2015 19:06:06 -0500 Subject: [PATCH 6/8] removed the old implementation of Subordinate --- datajoint/__init__.py | 4 ++-- datajoint/user_relations.py | 38 +------------------------------------ 2 files changed, 3 insertions(+), 39 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 5b2776a69..ccc0b03ab 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -17,7 +17,7 @@ 'config', 'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'Relation', 'schema', - 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub', + 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub' 'conn', 'kill'] @@ -55,7 +55,7 @@ class DataJointError(Exception): # ------------- flatten import hierarchy ------------------------- from .connection import conn, Connection from .relation import Relation -from .user_relations import Manual, Lookup, Imported, Computed, Sub, Subordinate +from .user_relations import Manual, Lookup, Imported, Computed, Sub from .relational_operand import Not from .heading import Heading from .schema import schema diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 65eb354c7..24f5751ce 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -83,40 +83,4 @@ def table_name(self): :returns: the table name of the table formatted for mysql. """ return "__" + from_camel_case(self.__class__.__name__) - - -class Subordinate: - """ - Mix-in to make computed tables subordinate. - This class is DEPRECATED and will be removed in a future version. Use dj.Sub instead. - """ - - @property - def populated_from(self): - """ - Overrides the `populate_from` property because subtables should not be populated - directly. - :return: None - """ - return None - - def _make_tuples(self, key): - """ - Overrides the `_make_tuples` property because subtables should not be populated - directly. Raises an error if this method is called (usually from populate of the - inheriting object). - :raises: NotImplementedError - """ - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') - - def progress(self): - """ - Overrides the `progress` method because subtables should not be populated directly. - """ - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') - - def populate(self, *args, **kwargs): - raise NotImplementedError( - 'This table is subordinate: it cannot be populated directly. Refer to its parent table.') \ No newline at end of file + \ No newline at end of file From 70896c679478a94eb843a677dc5a17dbd9a4c007 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sun, 23 Aug 2015 19:08:18 -0500 Subject: [PATCH 7/8] typo fixed --- datajoint/__init__.py | 2 +- datajoint/user_relations.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index ccc0b03ab..9c85b4e43 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -17,7 +17,7 @@ 'config', 'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'Relation', 'schema', - 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub' + 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub', 'conn', 'kill'] diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 24f5751ce..4d12a684d 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -83,4 +83,3 @@ def table_name(self): :returns: the table name of the table formatted for mysql. """ return "__" + from_camel_case(self.__class__.__name__) - \ No newline at end of file From fdb1bc213bfeb999d9465660bb3b9867e762568b Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 24 Aug 2015 16:15:45 -0500 Subject: [PATCH 8/8] renamed datajoint.Sub into datajoint.Part --- datajoint/__init__.py | 4 ++-- datajoint/schema.py | 19 +++++++++---------- datajoint/user_relations.py | 4 ++-- tests/schema.py | 17 ++++++++--------- tests/schema_simple.py | 10 ++++------ 5 files changed, 25 insertions(+), 29 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 9c85b4e43..aea688382 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -17,7 +17,7 @@ 'config', 'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'Relation', 'schema', - 'Manual', 'Lookup', 'Imported', 'Computed', 'Sub', + 'Manual', 'Lookup', 'Imported', 'Computed', 'Part', 'conn', 'kill'] @@ -55,7 +55,7 @@ class DataJointError(Exception): # ------------- flatten import hierarchy ------------------------- from .connection import conn, Connection from .relation import Relation -from .user_relations import Manual, Lookup, Imported, Computed, Sub +from .user_relations import Manual, Lookup, Imported, Computed, Part from .relational_operand import Not from .heading import Heading from .schema import schema diff --git a/datajoint/schema.py b/datajoint/schema.py index c6739d837..bdf985c61 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -4,7 +4,7 @@ from . import conn, DataJointError from .heading import Heading from .relation import Relation -from .user_relations import Sub +from .user_relations import Part logger = logging.getLogger(__name__) @@ -57,25 +57,24 @@ def process_relation_class(class_object, context): instance.heading # trigger table declaration instance._prepare() - if issubclass(cls, Sub): - raise DataJointError( - 'Subordinate relations need not be assigned to a schema directly') + if issubclass(cls, Part): + raise DataJointError('The schema decorator should not apply to part relations') process_relation_class(cls, context=self.context) # Process subordinate relations for name in (name for name in dir(cls) if not name.startswith('_')): - sub = getattr(cls, name) + part = getattr(cls, name) try: - is_sub = issubclass(sub, Sub) + is_sub = issubclass(part, Part) except TypeError: pass else: if is_sub: - sub._master = cls - process_relation_class(sub, context=dict(self.context, **{cls.__name__: cls})) - elif issubclass(sub, Relation): - raise DataJointError('Subordinate relations must subclass from datajoint.Sub') + part._master = cls + process_relation_class(part, context=dict(self.context, **{cls.__name__: cls})) + elif issubclass(part, Relation): + raise DataJointError('Part relations must subclass from datajoint.Part') return cls @property diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py index 4d12a684d..2e140f62f 100644 --- a/datajoint/user_relations.py +++ b/datajoint/user_relations.py @@ -8,13 +8,13 @@ from . import DataJointError -class Sub(Relation): +class Part(Relation): @property def master(self): if not hasattr(self, '_master'): raise DataJointError( - 'subordinate relations must be declared inside a base relation class') + 'Part relations must be declared inside a base relation class') return self._master @property diff --git a/tests/schema.py b/tests/schema.py index 2bc9d9dde..564b0fcd4 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -66,13 +66,12 @@ class Language(dj.Lookup): """ contents = [ - ('Fabian', 'English'), - ('Edgar', 'English'), - ('Dimitri', 'English'), - ('Dimitri', 'Ukrainian'), - ('Fabian', 'German'), - ('Edgar', 'Japanese'), - ] + ('Fabian', 'English'), + ('Edgar', 'English'), + ('Dimitri', 'English'), + ('Dimitri', 'Ukrainian'), + ('Fabian', 'German'), + ('Edgar', 'Japanese')] @schema @@ -136,7 +135,7 @@ class Ephys(dj.Imported): duration :double # (s) """ - class Channel(dj.Sub): + class Channel(dj.Part): definition = """ # subtable containing individual channels -> Ephys channel :tinyint unsigned # channel number within Ephys @@ -154,7 +153,7 @@ def _make_tuples(self, key): duration=np.minimum(2, random.expovariate(1))) self.insert1(row) number_samples = round(row['duration'] * row['sampling_frequency']) - sub = Ephys.Channel() + sub = self.Channel() for channel in range(2): sub.insert1( dict(key, diff --git a/tests/schema_simple.py b/tests/schema_simple.py index cf7eb4289..61fedbb66 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -29,7 +29,7 @@ class B(dj.Computed): n :smallint # number samples """ - class C(dj.Sub): + class C(dj.Part): definition = """ -> B id_c :int @@ -46,8 +46,7 @@ def _make_tuples(self, key): sigma = random.lognormvariate(0, 4) n = random.randint(0, 10) self.insert1(dict(key, mu=mu, sigma=sigma, n=n)) - for j in range(n): - sub.insert1(dict(key, id_c=j, value=random.normalvariate(mu, sigma))) + sub.insert((dict(key, id_c=j, value=random.normalvariate(mu, sigma)) for j in range(n))) @schema @@ -70,7 +69,7 @@ class D(dj.Computed): """ def _make_tuples(self, key): - # connect to random L + # make reference to a random tuple from L random.seed(str(key)) lookup = list(L().fetch.keys()) for i in range(4): @@ -86,7 +85,7 @@ class E(dj.Computed): -> L """ - class F(dj.Sub): + class F(dj.Part): definition = """ -> E id_f :int @@ -103,4 +102,3 @@ def _make_tuples(self, key): for i, ref in enumerate(references): if random.getrandbits(1): sub.insert1(dict(key, id_f=i, **ref)) -