From 3dc832ae77fab94e479b782215bbeb1cc083c365 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 08:27:31 +0100 Subject: [PATCH 01/12] separated tests --- test_autofit/test_aggregator.py | 45 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 959abb1e5..1825d01a8 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -89,7 +89,7 @@ def test_pickles(self, path_aggregator): ))[0]["name"] == "optimizer" -class TestOperations: +class TestFiltering: def test_not_contains(self, aggregator): predicate = ~(aggregator.pipeline.contains("1")) result = aggregator.filter( @@ -119,26 +119,6 @@ def test_rhs(self, aggregator): ) assert result.pipeline == ["pipeline1", "pipeline1"] - def test_attribute(self, aggregator): - assert list( - aggregator.values("pipeline") - ) == ["pipeline1", "pipeline1", "pipeline2"] - assert list( - aggregator.values("phase") - ) == ["phase1", "phase2", "phase2"] - assert list( - aggregator.values("dataset") - ) == ["dataset1", "dataset1", "dataset2"] - - def test_indexing(self, aggregator): - assert list( - aggregator[1:].values("pipeline") - ) == ["pipeline1", "pipeline2"] - assert list( - aggregator[-1:].values("pipeline") - ) == ["pipeline2"] - assert aggregator[0].pipeline == "pipeline1" - def test_filter_index(self, aggregator): assert list(aggregator.filter( aggregator.pipeline == "pipeline1" @@ -206,6 +186,29 @@ def test_filter_contains_directory(self, aggregator): ) assert len(result) == 1 + +class TestOperations: + + def test_attribute(self, aggregator): + assert list( + aggregator.values("pipeline") + ) == ["pipeline1", "pipeline1", "pipeline2"] + assert list( + aggregator.values("phase") + ) == ["phase1", "phase2", "phase2"] + assert list( + aggregator.values("dataset") + ) == ["dataset1", "dataset1", "dataset2"] + + def test_indexing(self, aggregator): + assert list( + aggregator[1:].values("pipeline") + ) == ["pipeline1", "pipeline2"] + assert list( + aggregator[-1:].values("pipeline") + ) == ["pipeline2"] + assert aggregator[0].pipeline == "pipeline1" + def test_group_by(self, aggregator): result = aggregator.group_by("pipeline") From 0e2000b87de54cf9b53d96addbebdc5d45f9ff79 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 08:32:44 +0100 Subject: [PATCH 02/12] or predicate for aggregator --- autofit/aggregator/predicate.py | 14 ++++++++++++++ test_autofit/test_aggregator.py | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 169a88eb7..9359721e8 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -82,6 +82,9 @@ def __invert__(self) -> "NotPredicate": self ) + def __or__(self, other) -> "OrPredicate": + return OrPredicate(self, other) + @abstractmethod def __call__(self, phase: PhaseOutput) -> bool: """ @@ -89,6 +92,17 @@ def __call__(self, phase: PhaseOutput) -> bool: """ +class CombinationPredicate(AbstractPredicate, ABC): + def __init__(self, one, two): + self.one = one + self.two = two + + +class OrPredicate(CombinationPredicate): + def __call__(self, phase: PhaseOutput): + return self.one(phase) or self.two(phase) + + class ComparisonPredicate(AbstractPredicate, ABC): def __init__( self, diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 1825d01a8..f91215d19 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -90,6 +90,18 @@ def test_pickles(self, path_aggregator): class TestFiltering: + def test_or(self, aggregator): + predicate_one = aggregator.directory.contains("one") + predicate_two = aggregator.directory.contains("two") + result = aggregator.filter( + predicate_one | predicate_two + ) + assert len(result) == 2 + assert result.directories == [ + "directory/number/one", + "directory/number/two" + ] + def test_not_contains(self, aggregator): predicate = ~(aggregator.pipeline.contains("1")) result = aggregator.filter( From 3273ee1de43f298e7000945e753df3686d59261d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 09:29:14 +0100 Subject: [PATCH 03/12] and predicate --- autofit/aggregator/predicate.py | 8 +++++++ test_autofit/test_aggregator.py | 40 ++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 9359721e8..19d0bb78d 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -85,6 +85,9 @@ def __invert__(self) -> "NotPredicate": def __or__(self, other) -> "OrPredicate": return OrPredicate(self, other) + def __and__(self, other) -> "AndPredicate": + return AndPredicate(self, other) + @abstractmethod def __call__(self, phase: PhaseOutput) -> bool: """ @@ -103,6 +106,11 @@ def __call__(self, phase: PhaseOutput): return self.one(phase) or self.two(phase) +class AndPredicate(CombinationPredicate): + def __call__(self, phase: PhaseOutput): + return self.one(phase) and self.two(phase) + + class ComparisonPredicate(AbstractPredicate, ABC): def __init__( self, diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index f91215d19..887290ae5 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -32,9 +32,24 @@ def output(self): def make_aggregator(): aggregator = af.Aggregator("") aggregator.phases = [ - MockPhaseOutput("directory/number/one", "pipeline1", "phase1", "dataset1"), - MockPhaseOutput("directory/number/two", "pipeline1", "phase2", "dataset1"), - MockPhaseOutput("directory/letter/a", "pipeline2", "phase2", "dataset2"), + MockPhaseOutput( + "directory/number/one", + "pipeline1", + "phase1", + "dataset1" + ), + MockPhaseOutput( + "directory/number/two", + "pipeline1", + "phase2", + "dataset1" + ), + MockPhaseOutput( + "directory/letter/a", + "pipeline2", + "phase2", + "dataset2" + ), ] return aggregator @@ -91,8 +106,12 @@ def test_pickles(self, path_aggregator): class TestFiltering: def test_or(self, aggregator): - predicate_one = aggregator.directory.contains("one") - predicate_two = aggregator.directory.contains("two") + predicate_one = aggregator.directory.contains( + "one" + ) + predicate_two = aggregator.directory.contains( + "two" + ) result = aggregator.filter( predicate_one | predicate_two ) @@ -102,6 +121,17 @@ def test_or(self, aggregator): "directory/number/two" ] + def test_and(self, aggregator): + predicate_one = aggregator.pipeline == "pipeline1" + predicate_two = aggregator.phase == "phase2" + result = aggregator.filter( + predicate_one & predicate_two + ) + assert len(result) == 1 + assert result.directories == [ + "directory/number/two" + ] + def test_not_contains(self, aggregator): predicate = ~(aggregator.pipeline.contains("1")) result = aggregator.filter( From d9032092025d1c546899db486beadd3886547681 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 09:30:52 +0100 Subject: [PATCH 04/12] docs --- autofit/aggregator/predicate.py | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 19d0bb78d..91fcba8d8 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -96,18 +96,55 @@ def __call__(self, phase: PhaseOutput) -> bool: class CombinationPredicate(AbstractPredicate, ABC): - def __init__(self, one, two): + def __init__( + self, + one: AbstractPredicate, + two: AbstractPredicate + ): + """ + Abstract predicate combining two other predicates. + + Parameters + ---------- + one + two + Child predicates + """ self.one = one self.two = two class OrPredicate(CombinationPredicate): def __call__(self, phase: PhaseOutput): + """ + The disjunction of two predicates. + + Parameters + ---------- + phase + An object representing the output of a given phase. + + Returns + ------- + True if either predicate is True for the phase + """ return self.one(phase) or self.two(phase) class AndPredicate(CombinationPredicate): def __call__(self, phase: PhaseOutput): + """ + The conjunction of two predicates. + + Parameters + ---------- + phase + An object representing the output of a given phase. + + Returns + ------- + True if both predicates are True for the phase + """ return self.one(phase) and self.two(phase) From 820b8ce232f99d41ad856d45d17a8ab5307d260f Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 10:20:58 +0100 Subject: [PATCH 05/12] equality with an attribute of a pickled object --- autofit/aggregator/__init__.py | 5 ++ autofit/aggregator/predicate.py | 46 +++++++++++------- test_autofit/test_aggregator.py | 8 +++ test_autofit/test_files/aggregator/phase.zip | Bin 1612 -> 2382 bytes .../test_files/aggregator/phase_completed.zip | Bin 1928 -> 2165 bytes test_autofit/tools/test_dataset.py | 2 +- 6 files changed, 43 insertions(+), 18 deletions(-) diff --git a/autofit/aggregator/__init__.py b/autofit/aggregator/__init__.py index ec029b3a4..fd98a8acc 100644 --- a/autofit/aggregator/__init__.py +++ b/autofit/aggregator/__init__.py @@ -1 +1,6 @@ from .aggregator import * + + +class PickledChild: + def __init__(self, age): + self.age = age diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 91fcba8d8..ba0c344d6 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -1,11 +1,11 @@ from abc import ABC, abstractmethod -from typing import List, Iterator +from typing import List, Iterator, Iterable from .phase_output import PhaseOutput class AttributePredicate: - def __init__(self, attribute: str): + def __init__(self, *path): """ Used to produce predicate objects for filtering in the aggregator. @@ -15,10 +15,10 @@ def __init__(self, attribute: str): Parameters ---------- - attribute - The name of the attribute this predicate relates to. + path + The names of the attribute this predicate relates to. """ - self.attribute = attribute + self.path = path def __eq__(self, value): """ @@ -26,10 +26,15 @@ def __eq__(self, value): the attribute of a phase. """ return EqualityPredicate( - self.attribute, + self.path, value ) + def __getattr__(self, item): + return AttributePredicate( + *self.path, item + ) + def __ne__(self, other): """ Create a predicate which asks whether the given value is not equal to @@ -43,7 +48,7 @@ def contains(self, value): the attribute of a phase. """ return ContainsPredicate( - self.attribute, + self.path, value ) @@ -151,7 +156,7 @@ def __call__(self, phase: PhaseOutput): class ComparisonPredicate(AbstractPredicate, ABC): def __init__( self, - attribute: str, + path: Iterable[str], value ): """ @@ -159,14 +164,23 @@ def __init__( Parameters ---------- - attribute - An attribute of a phase + path + An attribute path of a phase value A value to which the attribute is compared """ - self.attribute = attribute + self.path = path self.value = value + def value_for_phase(self, phase): + value = phase + for attribute in self.path: + value = getattr( + value, + attribute + ) + return value + class ContainsPredicate(ComparisonPredicate): def __call__( @@ -184,9 +198,8 @@ def __call__( True iff the value of the attribute of the phase contains the value associated with this predicate """ - return self.value in getattr( - phase, - self.attribute + return self.value in self.value_for_phase( + phase ) @@ -203,9 +216,8 @@ def __call__(self, phase): True iff the value of the attribute of the phase is equal to the value associated with this predicate """ - return getattr( - phase, - self.attribute + return self.value_for_phase( + phase ) == self.value diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 887290ae5..32e27812a 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -105,6 +105,14 @@ def test_pickles(self, path_aggregator): class TestFiltering: + def test_filter_pickle(self, path_aggregator): + predicate = path_aggregator.child.age == 17 + result = path_aggregator.filter( + predicate + ) + assert len(result) == 1 + assert list(result.values("child"))[0].age == 17 + def test_or(self, aggregator): predicate_one = aggregator.directory.contains( "one" diff --git a/test_autofit/test_files/aggregator/phase.zip b/test_autofit/test_files/aggregator/phase.zip index d8e9d314d81a460dd70b19d2cdbc83a457f388ae..ceeb6444ee0519d6964a926bb8fdc2a741bb9878 100644 GIT binary patch literal 2382 zcmWIWW@h1H00G&c{s1roO0Y2qFcf4Y7N_cmM({B3UYr)k1;k$&MHm8rdPEpFfH2e- zRS!R0kDg0#d~ivAQ7TL`(@BWtw==By4jBlzr1x@ro9JoYEnvt?a6QSngMl@C!S{=b zw*)td1O=!zRZjXE)>WS=&srhYWwQ#@L_)I^1Pgd+IR2l;^5dQ>L1|E z&XL8?k-G^PFf0-bVDCeMOc5=}IDy`ekN0(S_79GLhZ-p4V4;Rx8z|J^+E9ZWSu^$^ zm%`Mn7mqXalM_Uf(-V>sKKS~Cec%u42xwrMAi=CIAkDZ@TA=xdsKXhC;D-ta8kjBG zT38%siWnT?w}6A<^jUw;vlq3!boI_^YCdBETTuUf0;>Sfa6Y^ih`=pKNh|@T%@VzW%;fAG za4P$90%1dT!aCiQj)cU7gcIIpe05WL64;G`g}jBjgT1+Cc6|lgxQXqIv;xpJ5XS8# zA-IhYFA-}Wt5Nai;Gkl-ZQS$yBjtcLfiP~{#Nf8&7nEe?W>%#Z5$nSTEJmNXy^M;D zR|j>&ZPhp!EUN^x5rh%83ZSK1gddYLGILUpY=di(h6 zd-?k7oYa5Bb;9S|IZxlhb0^MtKGaINkfW2bMmMD^AuS=n!^dAY<=ZnYze~*E0Po~c zD7FPU#2Uph?A54ghz}kFxv3?IplF6AAKoR93o6Zk{zrP5=#WKiiEnoy}E;ggM_?;rZP7W^$71@HBaJP=)#2Jr8DPJ$t)8UO3iQ`wu-_P&M453F**riMVBql9 z5k$jE6ApL*9^ehqk0?u!OE!K6g#IOsCUE^o#ROB$DB8iZ0@qWceGCa9c4E}(=_%rs$w z1vFZz#{n<(kV7407;@Pof@0WIxMAoe90voeghMwCxoi`zcJft<(rP~29*0{17p(13(KsL((+ h2RT1UqL_0PXbzt80oh1aHc$rR0m62mM>ev8cmM!}e>MOB delta 446 zcmX>nbcTmFz?+$civa}Ks~acsN^>v1)*HvWq$TbvqsU}qCNt)eYrT_mnBu|o24<%6N?&{y_!Xj6JnrWS;u4tMmb54i69I#kpW>HNVc3&1#BNwcJdV# z8+M?5Ap0l3Vw9h3#@fydbk^hpteWglcTJXL*O@HC#yL5HiJz5&0pz+3?4py)*`?)x zekjODEKb!=Nh|?UC3*#!$=Nxnp%E};28@i8AF@k8oD9??$RQ0B(dCcvhmH=-?CQ)XD2R+%OH9VLY7~Sb5j2i*9o6<=RAE2&z(5q`A{q6LXJ+# z8r_txgtUYN4ewUaT0=#+IImGW+e6a=EZq2|j*^^m%ay^p*$mNq4F_|#y z1@%w9$s{KU@)QVzY()4NBrCwI2lg^hc5(uf{Nxm7ZLsc1%&VAzAu`#X#TXVOlPg&Q zz;b6;LP0?=S%oziOjogna{&2qTNby(O*UYWnf#U2MGzD)NKRvrne4@;D2^Hw9I&96 ztiz_w_-Aqio4h#C>2bVETH*q{8JR?xAyETzXdj!jMlv%4LwvliqqBc-gnnoQ2LsGs t=!!40Ni)ux?7%L=jihR_2)j1pL!hXZ9KsS-Hc-TI1K|vy;m27(JOETPtdRf! delta 338 zcmew=(7``hmPu-&lmz?oYrS!EuJul|bYU(5QVp44>PlAhei7zbfWP5#4JJoz=V zz+^FI1(48WOJ)=1fU=Is17Xt`@WS2ARfh_^bPQJtJ1=gv_ zvTAY-o51A9ENUc|4dgX0Aar78U@%|-@c=KZZ)pGk diff --git a/test_autofit/tools/test_dataset.py b/test_autofit/tools/test_dataset.py index 7370ab3e4..aeaf4affc 100644 --- a/test_autofit/tools/test_dataset.py +++ b/test_autofit/tools/test_dataset.py @@ -4,4 +4,4 @@ def test_save_and_load(): dataset = MockDataset() dataset.save("/tmp") - assert dataset.name == MockDataset.load(f"/tmp/{dataset.name}.pickle").name + assert dataset.name == MockDataset.load(f"/tmp/dataset.pickle").name From 35a5bec6e9ae6948437c2b4b8537b9ad22b1089f Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 10:23:42 +0100 Subject: [PATCH 06/12] docs --- autofit/aggregator/predicate.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index ba0c344d6..2b506dea8 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -16,7 +16,9 @@ def __init__(self, *path): Parameters ---------- path - The names of the attribute this predicate relates to. + A series of names of attributes that can be used to get a value. + For example, (mask, pixel_size) would get the pixel size of a mask + when evaluated for a given phase. """ self.path = path @@ -30,7 +32,10 @@ def __eq__(self, value): value ) - def __getattr__(self, item): + def __getattr__(self, item: str) -> "AttributePredicate": + """ + Adds another item to the path + """ return AttributePredicate( *self.path, item ) @@ -87,10 +92,18 @@ def __invert__(self) -> "NotPredicate": self ) - def __or__(self, other) -> "OrPredicate": + def __or__(self, other: "AbstractPredicate") -> "OrPredicate": + """ + Returns a predicate that is true if either predicate is true + for a given phase. + """ return OrPredicate(self, other) - def __and__(self, other) -> "AndPredicate": + def __and__(self, other: "AbstractPredicate") -> "AndPredicate": + """ + Returns a predicate that is true if both predicates are true + for a given phase. + """ return AndPredicate(self, other) @abstractmethod From 824819ad42c41211a33e2f8d39356049ee51976d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 10:44:37 +0100 Subject: [PATCH 07/12] inequalities --- autofit/aggregator/predicate.py | 24 +++++++++++++ test_autofit/test_aggregator.py | 63 +++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 2b506dea8..676b25438 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -40,6 +40,16 @@ def __getattr__(self, item: str) -> "AttributePredicate": *self.path, item ) + def __gt__(self, other): + return GreaterThanPredicate( + self.path, other + ) + + def __lt__(self, other): + return LessThanPredicate( + self.path, other + ) + def __ne__(self, other): """ Create a predicate which asks whether the given value is not equal to @@ -195,6 +205,20 @@ def value_for_phase(self, phase): return value +class GreaterThanPredicate(ComparisonPredicate): + def __call__(self, phase): + return self.value_for_phase( + phase + ) > self.value + + +class LessThanPredicate(ComparisonPredicate): + def __call__(self, phase): + return self.value_for_phase( + phase + ) < self.value + + class ContainsPredicate(ComparisonPredicate): def __call__( self, diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 32e27812a..4006d423b 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -104,8 +104,25 @@ def test_pickles(self, path_aggregator): ))[0]["name"] == "optimizer" -class TestFiltering: - def test_filter_pickle(self, path_aggregator): +@pytest.fixture( + name="ages_for_predicate" +) +def make_ages_for_predicate(path_aggregator): + def ages_for_predicate( + predicate + ): + result = path_aggregator.filter( + predicate + ) + return [child.age for child in result.values( + "child" + )] + + return ages_for_predicate + + +class TestNumericalFiltering: + def test_equality(self, path_aggregator): predicate = path_aggregator.child.age == 17 result = path_aggregator.filter( predicate @@ -113,6 +130,48 @@ def test_filter_pickle(self, path_aggregator): assert len(result) == 1 assert list(result.values("child"))[0].age == 17 + def test_greater_than( + self, + ages_for_predicate, + path_aggregator + ): + ages = ages_for_predicate( + path_aggregator.child.age > 10 + ) + assert ages == [17] + + def test_less_than( + self, + ages_for_predicate, + path_aggregator + ): + ages = ages_for_predicate( + path_aggregator.child.age < 11 + ) + assert ages == [10] + + def test_greater_than_rhs( + self, + ages_for_predicate, + path_aggregator + ): + ages = ages_for_predicate( + 10 < path_aggregator.child.age + ) + assert ages == [17] + + def test_less_than_rhs( + self, + ages_for_predicate, + path_aggregator + ): + ages = ages_for_predicate( + 11 > path_aggregator.child.age + ) + assert ages == [10] + + +class TestFiltering: def test_or(self, aggregator): predicate_one = aggregator.directory.contains( "one" From 6595872ec8a70393b8a0e609f7b99c5b3ce9a852 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 10:53:45 +0100 Subject: [PATCH 08/12] docs --- autofit/aggregator/predicate.py | 41 +++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 676b25438..a551eaf00 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -41,11 +41,19 @@ def __getattr__(self, item: str) -> "AttributePredicate": ) def __gt__(self, other): + """ + Is the value of this attribute for a given phase greater than some + other value? + """ return GreaterThanPredicate( self.path, other ) def __lt__(self, other): + """ + Is the value of this attribute for a given phase less than some + other value? + """ return LessThanPredicate( self.path, other ) @@ -206,14 +214,43 @@ def value_for_phase(self, phase): class GreaterThanPredicate(ComparisonPredicate): - def __call__(self, phase): + def __call__( + self, + phase: PhaseOutput + ) -> bool: + """ + Parameters + ---------- + phase + An object representing the output of a given phase. + + Returns + ------- + True iff the value of the attribute of the phase is greater than + the value associated with this predicate + """ + return self.value_for_phase( phase ) > self.value class LessThanPredicate(ComparisonPredicate): - def __call__(self, phase): + def __call__( + self, + phase: PhaseOutput + ) -> bool: + """ + Parameters + ---------- + phase + An object representing the output of a given phase. + + Returns + ------- + True iff the value of the attribute of the phase is less than + the value associated with this predicate + """ return self.value_for_phase( phase ) < self.value From 8dd1fd08de2be530bd68a1c23878a1d4a110c41d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 10:55:53 +0100 Subject: [PATCH 09/12] docs --- autofit/aggregator/predicate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index a551eaf00..986191d5a 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -203,7 +203,14 @@ def __init__( self.path = path self.value = value - def value_for_phase(self, phase): + def value_for_phase( + self, + phase: PhaseOutput + ): + """ + Recurse the phase output by iterating the attributes in the path + and getting a value for each attribute. + """ value = phase for attribute in self.path: value = getattr( From 5b05eab3bd54492ad6000d404ed5771f46dcbbb1 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 11:19:42 +0100 Subject: [PATCH 10/12] handling comparison of aggregator attributes --- .gitignore | 3 +- autofit/aggregator/predicate.py | 76 +++++++++++++++++++++------------ test_autofit/test_aggregator.py | 7 +++ 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 90e30f1fe..17b55f6a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +__MACOSX *.swp test/autofit/test_fit # Byte-compiled / optimized / DLL files @@ -114,4 +115,4 @@ test_autofit/optimize/test_fit/ test_autofit/test_files/text/psycopg2-binary==2.8.1 test_autofit/test_files/text/ fit/test_autofit/optimize/test_fit -.DS_Store +*.DS_Store diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 986191d5a..11626bfbb 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -22,13 +22,29 @@ def __init__(self, *path): """ self.path = path + def value_for_phase( + self, + phase: PhaseOutput + ): + """ + Recurse the phase output by iterating the attributes in the path + and getting a value for each attribute. + """ + value = phase + for attribute in self.path: + value = getattr( + value, + attribute + ) + return value + def __eq__(self, value): """ Create a predicate which asks whether the given value is equal to the attribute of a phase. """ return EqualityPredicate( - self.path, + self, value ) @@ -46,7 +62,7 @@ def __gt__(self, other): other value? """ return GreaterThanPredicate( - self.path, other + self, other ) def __lt__(self, other): @@ -55,7 +71,7 @@ def __lt__(self, other): other value? """ return LessThanPredicate( - self.path, other + self, other ) def __ne__(self, other): @@ -71,7 +87,7 @@ def contains(self, value): the attribute of a phase. """ return ContainsPredicate( - self.path, + self, value ) @@ -187,7 +203,7 @@ def __call__(self, phase: PhaseOutput): class ComparisonPredicate(AbstractPredicate, ABC): def __init__( self, - path: Iterable[str], + attribute_predicate: AttributePredicate, value ): """ @@ -195,29 +211,23 @@ def __init__( Parameters ---------- - path + attribute_predicate An attribute path of a phase value A value to which the attribute is compared """ - self.path = path - self.value = value + self.attribute_predicate = attribute_predicate + self._value = value - def value_for_phase( + def value( self, - phase: PhaseOutput + phase ): - """ - Recurse the phase output by iterating the attributes in the path - and getting a value for each attribute. - """ - value = phase - for attribute in self.path: - value = getattr( - value, - attribute + if isinstance(self._value, AttributePredicate): + return self._value.value_for_phase( + phase ) - return value + return self._value class GreaterThanPredicate(ComparisonPredicate): @@ -237,9 +247,11 @@ def __call__( the value associated with this predicate """ - return self.value_for_phase( + return self.attribute_predicate.value_for_phase( phase - ) > self.value + ) > self.value( + phase + ) class LessThanPredicate(ComparisonPredicate): @@ -258,9 +270,11 @@ def __call__( True iff the value of the attribute of the phase is less than the value associated with this predicate """ - return self.value_for_phase( + return self.attribute_predicate.value_for_phase( phase - ) < self.value + ) < self.value( + phase + ) class ContainsPredicate(ComparisonPredicate): @@ -279,7 +293,9 @@ def __call__( True iff the value of the attribute of the phase contains the value associated with this predicate """ - return self.value in self.value_for_phase( + return self.value( + phase + ) in self.attribute_predicate.value_for_phase( phase ) @@ -297,9 +313,15 @@ def __call__(self, phase): True iff the value of the attribute of the phase is equal to the value associated with this predicate """ - return self.value_for_phase( + try: + value = self.value( + phase + ) + except AttributeError: + value = self.value + return self.attribute_predicate.value_for_phase( phase - ) == self.value + ) == value class NotPredicate(AbstractPredicate): diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 4006d423b..1ca1b4c04 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -170,6 +170,13 @@ def test_less_than_rhs( ) assert ages == [10] + def test_aggregator_to_aggregator(self, path_aggregator): + predicate = path_aggregator.child.age == path_aggregator.child.age + assert len(path_aggregator.filter(predicate)) == 2 + + predicate = path_aggregator.child.age > path_aggregator.child.age + assert len(path_aggregator.filter(predicate)) == 0 + class TestFiltering: def test_or(self, aggregator): From f557bc2dae84b3db30f60937cb7ac56b6e171cd6 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 11:21:51 +0100 Subject: [PATCH 11/12] ge and le --- autofit/aggregator/predicate.py | 14 +++++++++++++- test_autofit/test_aggregator.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/autofit/aggregator/predicate.py b/autofit/aggregator/predicate.py index 11626bfbb..870ce2b46 100644 --- a/autofit/aggregator/predicate.py +++ b/autofit/aggregator/predicate.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Iterator, Iterable +from typing import List, Iterator from .phase_output import PhaseOutput @@ -48,6 +48,18 @@ def __eq__(self, value): value ) + def __le__(self, other): + return OrPredicate( + self == other, + self < other + ) + + def __ge__(self, other): + return OrPredicate( + self == other, + self > other + ) + def __getattr__(self, item: str) -> "AttributePredicate": """ Adds another item to the path diff --git a/test_autofit/test_aggregator.py b/test_autofit/test_aggregator.py index 1ca1b4c04..22119bb89 100644 --- a/test_autofit/test_aggregator.py +++ b/test_autofit/test_aggregator.py @@ -140,6 +140,16 @@ def test_greater_than( ) assert ages == [17] + def test_greater_than_equal( + self, + ages_for_predicate, + path_aggregator + ): + ages = ages_for_predicate( + path_aggregator.child.age >= 10 + ) + assert set(ages) == {10, 17} + def test_less_than( self, ages_for_predicate, From b22e4042be661d502dd7b791538ad63721d9ce9c Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 15 Apr 2020 11:37:52 +0100 Subject: [PATCH 12/12] moved pickles to their own folder --- autofit/aggregator/phase_output.py | 16 ++++++++++------ autofit/optimize/non_linear/paths.py | 12 ++++++++---- autofit/tools/phase.py | 15 ++++----------- test_autofit/test_files/aggregator/phase.zip | Bin 2382 -> 2578 bytes .../test_files/aggregator/phase_completed.zip | Bin 2165 -> 2974 bytes 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/autofit/aggregator/phase_output.py b/autofit/aggregator/phase_output.py index f1e2ba219..7985132f5 100644 --- a/autofit/aggregator/phase_output.py +++ b/autofit/aggregator/phase_output.py @@ -35,6 +35,10 @@ def __init__(self, directory: str): ] self.__dict__.update({pair[0]: pair[1] for pair in pairs}) + @property + def pickle_path(self): + return f"{self.directory}/pickles" + @property def output(self) -> AbstractOutput: """ @@ -58,7 +62,7 @@ def mask(self): A pickled mask object """ with open( - os.path.join(self.directory, "mask.pickle"), "rb" + os.path.join(self.pickle_path, "mask.pickle"), "rb" ) as f: return dill.load(f) @@ -69,7 +73,7 @@ def __getattr__(self, item): dataset.pickle, meta_dataset.pickle etc. """ with open( - os.path.join(self.directory, f"{item}.pickle"), "rb" + os.path.join(self.pickle_path, f"{item}.pickle"), "rb" ) as f: return pickle.load(f) @@ -86,17 +90,17 @@ def optimizer(self) -> autofit.optimize.non_linear.non_linear.NonLinearOptimizer The optimizer object that was used in this phase """ if self.__optimizer is None: - with open(os.path.join(self.directory, "optimizer.pickle"), "r+b") as f: + with open(os.path.join(self.pickle_path, "optimizer.pickle"), "r+b") as f: self.__optimizer = pickle.loads(f.read()) return self.__optimizer @property - def model(self) -> autofit.optimize.non_linear.non_linear.NonLinearOptimizer: + def model(self): """ - The optimizer object that was used in this phase + The model that was used in this phase """ if self.__model is None: - with open(os.path.join(self.directory, "model.pickle"), "r+b") as f: + with open(os.path.join(self.pickle_path, "model.pickle"), "r+b") as f: self.__model = pickle.loads(f.read()) return self.__model diff --git a/autofit/optimize/non_linear/paths.py b/autofit/optimize/non_linear/paths.py index 01a4d1d64..5474becda 100644 --- a/autofit/optimize/non_linear/paths.py +++ b/autofit/optimize/non_linear/paths.py @@ -171,23 +171,27 @@ def pdf_path(self) -> str: """ return "{}pdf/".format(self.image_path) + @property + @make_path + def pickle_path(self) -> str: + return f"{self.make_path()}/pickles" + def make_optimizer_pickle_path(self) -> str: """ Create the path at which the optimizer pickle should be saved """ - return "{}/optimizer.pickle".format(self.make_path()) + return f"{self.pickle_path}/optimizer.pickle" def make_model_pickle_path(self): """ Create the path at which the model pickle should be saved """ - return "{}/model.pickle".format(self.make_path()) + return f"{self.pickle_path}/model.pickle" @make_path def make_path(self) -> str: """ - Create the path to the folder at which the metadata and optimizer pickle should - be saved + Create the path to the folder at which the metadata should be saved """ return "{}/{}/{}/{}/".format( conf.instance.output_path, self.phase_path, self.phase_name, self.phase_tag diff --git a/autofit/tools/phase.py b/autofit/tools/phase.py index f84c888c5..fe6800e13 100644 --- a/autofit/tools/phase.py +++ b/autofit/tools/phase.py @@ -82,26 +82,19 @@ def save_dataset(self, dataset): """ Save the dataset associated with the phase """ - - name = "dataset" - - if hasattr(dataset, "name"): - if dataset.name is not None: - name = dataset.name - - with open("{}/{}.pickle".format(self.paths.make_path(), name), "wb") as f: + with open(f"{self.paths.pickle_path}/dataset.pickle", "wb") as f: pickle.dump(dataset, f) def save_mask(self, mask): """ Save the mask associated with the phase """ - with open("{}/mask.pickle".format(self.paths.make_path()), "wb") as f: + with open(f"{self.paths.pickle_path}/mask.pickle", "wb") as f: dill.dump(mask, f) def save_meta_dataset(self, meta_dataset): with open( - f"{self.paths.phase_output_path}/meta_dataset.pickle", + f"{self.paths.pickle_path}/meta_dataset.pickle", "wb+" ) as f: pickle.dump( @@ -110,7 +103,7 @@ def save_meta_dataset(self, meta_dataset): def save_phase_attributes(self, phase_attributes): with open( - f"{self.paths.phase_output_path}/phase_attributes.pickle", + f"{self.paths.pickle_path}/phase_attributes.pickle", "wb+" ) as f: pickle.dump( diff --git a/test_autofit/test_files/aggregator/phase.zip b/test_autofit/test_files/aggregator/phase.zip index ceeb6444ee0519d6964a926bb8fdc2a741bb9878..e935990a02c19d17dd8772e6ee76f368da4fc5ff 100644 GIT binary patch delta 983 zcmX>nG)aUfz?+$civa|*Vnf}MHoPGA`Bcrs2elUL6z+rQ2g7Y ziP@K!R>e%VXOssk25EpPh8P5tEMru~kc62uc{8H{)TqfX85@|cJ)WG+q*o7fM1KI- zHBf?&L4ctkBe6JDzaTR?J14bRKQw}e0lOowRW+jMmBG-Pl2`&XszeWB7RaP)K-X_s z+!FV7@-}9X$uF2JRirR<<>sfP=Ah`Ayq;Nlas;!vh8%|C{DP9q+{~)fB6JO5SWHgN z$jnJWRjA0)4hoS~ENSc@xA87%nf!`TezF;BJCk$wz#`A#t>aWC z28_7lfX5xO^FcAUiS3NE0#G*y%P=6!S<-kBWR4f#JZqI^lJ8Im6wN2+R xkhlU@n44OX2=ZrW1X%JuhXIoz>tsPrc{ymp0LCRN8_LK=r&$34*Ka`RJCbI=vYOkT=trXU7alwVMi znVVUaT7<4pda@jgHIiw`8JRgLNQ&NHm|V-!4sskDYZ{2kV_nT88#Gy(O-l?GMgiW8 zOrp$ie?_wygQb{GPK$#`!NX$-n=V)iq!Uf*IaC)y3T~hQyB#c`O4!{&j#$qjHd&g3 zM@NK#2OMWW!eL3{R3-)-!Nvg!HlQXm?)mol{8*)x_5@CHkm=%E15()M!ovi3%3Dm8m6(pr{Uw;9vlHZ!xC<^Hmmx h$$qRd3Lt@LalA`ffC0wJ2J$Tr5Vix&-pC5#0RVonf} zMHm8rxIl2!XiaFa+-fNE3aq_1IJ^_epM3^zXPh;QIgIM{1!H|d60%Q~}jiY)?8 z9kt2pO74_NJ9Wq!&F61jS5hoJpWUh~*(mmfm-+UM@e7pazVH#>b?NL`=hG5)J=)U3 zU52d@3txw?*Ew>(py27x-}Z^7|G5?GB)T7$%6#BsInB4e?Dea*uig$H=C>}-%UP&> z_s%X3j*X)J0p9E!SqvSyn}C7EBEbL-2uLulL<>evU_ivj`#L)N2S>oe4-|&5@WZYR z6n=1RxS|2s80^uYg3lPec$_hkoFJNYdfpe8vQ}s{Z=~Rso1*}HJpM!&n;r4US z_m7kV+6}_E?bpC#e||wpW^QIxY7w!a@PNhWGq;yfvGMAlZn!%%4hG9A0o?$?2zRIl zp%zT|!XY^$GbaVj4N?~&nJqiP&re79r2lE1i{3u|`d+^NIw$oXah>ovch1wd@Z5tkF&BN=Qpc@bK~1P5CCK<9CS}98KbPEWX$Ron{Sn9wfnem35$`Z7D3l zqz5lTLCIszwca=ohLr%2tjk`Fnn$oD+}zZXL{M^rB{JS6knG{@spF~Rr`vfRn8ddF zHWq-L`+-USg&@$WA}FDWZjCa~xi||Ny?9(U;w~UQx+$=Ils6V&m2q}zTHrIGPuW4I ze_0Es*17Ws6z0vDzj}H^z=X-G#3w}sKAJUs&iqL;r}G7tOLnkINGuV^DiZ4U_UaA} z4ifSXn#$bd*yZJYsYG`TZ?A6;&skkvAB*W*jx@FeSBGmiWq^IjwtA9AE6|6J!Ml9E-8AG7}cEpwb^XzCjK}E|Jwx z9Lmd#=}?%7=rsYlixB~VT;iez^F%!6qn7fxO-C+AHBekH$b#GTm}NM28$jhTa#4ut z{+W1eKrNjirh`gnXo^%8V7@Am4G&ZumXx9bFmnL z866;lrp0Yp+!7by&B!Fm42%{DK4fpmGELsdW#S2nDe*fNUu=Q)SOYPtgHxF>9gG@p z91K#xpa-!qx6( xmtmU4JXwcBnmaTCu4=Ltw>Hy5kg)7z1vV}PWUE-&fIeX02ErLYJC3t}cmORA?MeUu