Merge pull request #425 from uwescience/bmyerz/improve_testing

Fix some testing issues

c_test_environment/grappalang_myrial_tests.py

@@ -19,16 +19,18 @@
 import logging
 logging.basicConfig(level=logging.DEBUG)
 
+def is_skipping():
+    return not ('RACO_GRAPPA_TESTS' in os.environ
+                and int(os.environ['RACO_GRAPPA_TESTS']) == 1)
 
 def raise_skip_test(query=None):
-  if 'RACO_GRAPPA_TESTS' in os.environ:
-    if int(os.environ['RACO_GRAPPA_TESTS']) == 1:
-        return
+     if not is_skipping():
+        return None
 
-  if query is not None:
-    raise SkipTest(query)
-  else:
-    raise SkipTest()
+     if query is not None:
+        raise SkipTest(query)
+     else:
+        raise SkipTest()
 
 
 class MyriaLGrappaTest(MyriaLPlatformTestHarness, MyriaLPlatformTests):
@@ -60,18 +62,19 @@ def check(self, query, name, **kwargs):
         with open(fname, 'w') as f:
             f.write(code)
 
+        #raise Exception()
         raise_skip_test(query)
 
         with Chdir("c_test_environment") as d:
             checkquery(name, GrappalangRunner())
 
     def setUp(self):
-        raise_skip_test()
         super(MyriaLGrappaTest, self).setUp()
-        with Chdir("c_test_environment") as d:
-            targetpath = os.path.join(os.environ.copy()['GRAPPA_HOME'], 'build/Make+Release/applications/join')
-            if need_generate(targetpath):
-                generate_default(targetpath)
+        if not is_skipping():
+            with Chdir("c_test_environment") as d:
+                targetpath = os.path.join(os.environ.copy()['GRAPPA_HOME'], 'build/Make+Release/applications/join')
+                if need_generate(targetpath):
+                    generate_default(targetpath)
 
     def _uda_def(self):
         uda_def_path = os.path.join("c_test_environment", "testqueries", "argmax.myl")

raco/language/clang.py

@@ -574,9 +574,6 @@ def opt_rules(self, **kwargs):
         # rules.FreeMemory()
         # ]
 
-        # disable specified rules
-        rules.Rule.set_global_rule_flags(*kwargs.keys())
-
         # sequence that works for myrial
         rule_grps_sequence = [
             rules.remove_trivial_sequences,
@@ -591,4 +588,10 @@ def opt_rules(self, **kwargs):
         if kwargs.get('SwapJoinSides'):
             rule_grps_sequence.insert(0, [rules.SwapJoinSides()])
 
-        return list(itertools.chain(*rule_grps_sequence))
+        # flatten the rules lists
+        rule_list = list(itertools.chain(*rule_grps_sequence))
+
+        # disable specified rules
+        rules.Rule.apply_disable_flags(rule_list, *kwargs.keys())
+
+        return rule_list

raco/language/clangcommon.py

@@ -545,6 +545,7 @@ class BreakHashJoinConjunction(rules.Rule):
     def __init__(self, select_clazz, join_clazz):
         self.select_clazz = select_clazz
         self.join_clazz = join_clazz
+        super(BreakHashJoinConjunction, self).__init__()
 
     def fire(self, expr):
         if isinstance(expr, self.join_clazz) \
@@ -616,6 +617,7 @@ def __init__(self, emit_print, subclass):
         assert issubclass(subclass, CBaseStore), \
             "%s is not a subclass of %s" % (subclass, CBaseStore)
         self.subclass = subclass
+        super(StoreToBaseCStore, self).__init__()
 
     def fire(self, expr):
         if isinstance(expr, algebra.Store):

raco/language/grappalang.py

@@ -1059,9 +1059,6 @@ def opt_rules(self, **kwargs):
         # rules.FreeMemory()
         # ]
 
-        # disable specified rules
-        rules.Rule.set_global_rule_flags(*kwargs.keys())
-
         join_type = kwargs.get('join_type', GrappaHashJoin)
 
         # sequence that works for myrial
@@ -1077,4 +1074,10 @@ def opt_rules(self, **kwargs):
         if kwargs.get('SwapJoinSides'):
             rule_grps_sequence.insert(0, [rules.SwapJoinSides()])
 
-        return list(itertools.chain(*rule_grps_sequence))
+        # flatten the rules lists
+        rule_list = list(itertools.chain(*rule_grps_sequence))
+
+        # disable specified rules
+        rules.Rule.apply_disable_flags(rule_list, *kwargs.keys())
+
+        return rule_list

raco/language/myrialang.py

@@ -936,6 +936,7 @@ class HCShuffleBeforeNaryJoin(rules.Rule):
     def __init__(self, catalog):
         assert isinstance(catalog, Catalog)
         self.catalog = catalog
+        super(HCShuffleBeforeNaryJoin, self).__init__()
 
     @staticmethod
     def reversed_index(child_schemes, conditions):
@@ -1324,6 +1325,7 @@ def fire(self, op):
 class PushIntoSQL(rules.Rule):
     def __init__(self, dialect=None):
         self.dialect = dialect or postgresql.dialect()
+        super(PushIntoSQL, self).__init__()
 
     def fire(self, expr):
         if isinstance(expr, (algebra.Scan, algebra.ScanTemp)):
@@ -1442,6 +1444,7 @@ class GetCardinalities(rules.Rule):
     def __init__(self, catalog):
         assert isinstance(catalog, Catalog)
         self.catalog = catalog
+        super(GetCardinalities, self).__init__()
 
     def fire(self, expr):
         # if not Zeroary operator, who cares?
@@ -1535,9 +1538,6 @@ class MyriaAlgebra(Algebra):
 class MyriaLeftDeepTreeAlgebra(MyriaAlgebra):
     """Myria physical algebra using left deep tree pipeline and 1-D shuffle"""
     def opt_rules(self, **kwargs):
-        # disable specified rules
-        rules.Rule.set_global_rule_flags(*kwargs.keys())
-
         opt_grps_sequence = [
             rules.remove_trivial_sequences,
             [
@@ -1572,7 +1572,14 @@ def opt_rules(self, **kwargs):
         compile_grps_sequence.append([BreakSplit()])
 
         rule_grps_sequence = opt_grps_sequence + compile_grps_sequence
-        return list(itertools.chain(*rule_grps_sequence))
+
+        # flatten the rules lists
+        rule_list = list(itertools.chain(*rule_grps_sequence))
+
+        # disable specified rules
+        rules.Rule.apply_disable_flags(rule_list, *kwargs.keys())
+
+        return rule_list
 
 
 class MyriaHyperCubeAlgebra(MyriaAlgebra):
@@ -1624,7 +1631,14 @@ def opt_rules(self, **kwargs):
         compile_grps_sequence.append([BreakSplit()])
 
         rule_grps_sequence = opt_grps_sequence + compile_grps_sequence
-        return list(itertools.chain(*rule_grps_sequence))
+
+        # flatten the rules lists
+        rule_list = list(itertools.chain(*rule_grps_sequence))
+
+        # disable specified rules
+        rules.Rule.apply_disable_flags(rule_list, *kwargs.keys())
+
+        return rule_list
 
     def __init__(self, catalog=None):
         self.catalog = catalog

raco/rules.py

@@ -17,24 +17,29 @@ class Rule(object):
 
     __metaclass__ = ABCMeta
 
-    _disabled_rules = set()
     _flag_pattern = re.compile(r'no_([A-Za-z_]+)')  # e.g., no_MergeSelects
 
+    def __init__(self):
+        self._disabled = False
+
     def __call__(self, expr):
-        # if the rule is in the set of disabled, then don't allow it to fire
-        if self.__class__.__name__ in self._disabled_rules:
+        if self._disabled:
             return expr
         else:
             return self.fire(expr)
 
     @classmethod
-    def set_global_rule_flags(cls, *args):
+    def apply_disable_flags(cls, rule_list, *args):
+        disabled_rules = set()
         # Automatically create a flag to disable any rule by name
         # e.g., to disable MergeSelects, pass the arg "no_MergeSelects"
         for a in args:
             mat = re.match(cls._flag_pattern, a)
             if mat:
-                cls._disabled_rules.add(mat.group(1))
+                disabled_rules.add(mat.group(1))
+
+        for r in rule_list:
+            r._disabled = r.__class__.__name__ in disabled_rules
 
     @abstractmethod
     def fire(self, expr):
@@ -74,6 +79,7 @@ class OneToOne(Rule):
     def __init__(self, opfrom, opto):
         self.opfrom = opfrom
         self.opto = opto
+        super(OneToOne, self).__init__()
 
     def fire(self, expr):
         if isinstance(expr, self.opfrom):