Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Hail] Python common subexpression elimination #7009

Merged
merged 50 commits into from Sep 17, 2019
Merged
Show file tree
Hide file tree
Changes from 47 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
62e5680
New CSE IR renrerer in Python
patrick-schultz Jul 24, 2019
689ca40
works with agg bindings
patrick-schultz Jul 29, 2019
ed133da
some refactoring
patrick-schultz Jul 31, 2019
3c392e6
finally fully working?
patrick-schultz Aug 1, 2019
9e788d3
loop -> recursion
patrick-schultz Aug 1, 2019
c2e9f92
CPS transform
patrick-schultz Aug 1, 2019
6619b17
inline recur
patrick-schultz Aug 1, 2019
28f23e3
defunctionalization
patrick-schultz Aug 1, 2019
144a1bd
recursion -> loop
patrick-schultz Aug 1, 2019
e0c76ad
cleaning up
patrick-schultz Aug 2, 2019
e12b7e5
some refactoring
patrick-schultz Aug 7, 2019
ed6ce81
more refactoring
patrick-schultz Aug 7, 2019
654f219
second pass: loop -> recursion
patrick-schultz Aug 7, 2019
f10306e
CPS
patrick-schultz Aug 8, 2019
8460050
factor out args into state object
patrick-schultz Aug 8, 2019
16eb524
inline print_renderable
patrick-schultz Aug 8, 2019
d196b2e
inline print
patrick-schultz Aug 8, 2019
3db71db
wip
patrick-schultz Aug 8, 2019
55c32ba
defunctionalize
patrick-schultz Aug 9, 2019
50ff007
recursion -> loop
patrick-schultz Aug 9, 2019
6ca6b23
linkedlist -> stack
patrick-schultz Aug 12, 2019
d2df592
refactoring
patrick-schultz Aug 12, 2019
1cccb05
refactoring, bugfixes
patrick-schultz Aug 13, 2019
fb6e986
bugfixes
patrick-schultz Aug 14, 2019
427d8c7
wip
patrick-schultz Aug 14, 2019
6aa0ba1
refactoring
patrick-schultz Aug 22, 2019
3d3671f
refactoring
patrick-schultz Aug 22, 2019
636aa5e
refactoring, documenting
patrick-schultz Aug 26, 2019
f64ac3e
refactoring
patrick-schultz Aug 26, 2019
3842f38
refactoring
patrick-schultz Aug 28, 2019
92c1f6b
refactoring
patrick-schultz Aug 28, 2019
2786aec
use namedtuple and __slots__
patrick-schultz Aug 28, 2019
0e54c39
[hail] Fix performance problems in aggregators by binding arguments
tpoterba Aug 29, 2019
e3be663
fix let lifting bug
tpoterba Aug 30, 2019
baa5251
bump!
tpoterba Aug 30, 2019
367819d
enable AggLets, other refactoring
patrick-schultz Sep 4, 2019
6320d30
put back old renderer
patrick-schultz Sep 4, 2019
f6b74c5
Merge branch 'tim-agglet-fix' into python-cse
patrick-schultz Sep 4, 2019
07e37a3
fixes
patrick-schultz Sep 5, 2019
8c9a83a
Squashed commit of the following:
patrick-schultz Sep 5, 2019
eb1225f
make _compute_type use bindings
patrick-schultz Sep 5, 2019
aa5f481
Merge branch 'python-binding' into python-cse
patrick-schultz Sep 5, 2019
705f3c3
wip refactoring
patrick-schultz Sep 5, 2019
538c1be
refactoring/documenting
patrick-schultz Sep 6, 2019
742ca62
Merge branch 'master' into python-cse
patrick-schultz Sep 10, 2019
bb2ce51
fix
patrick-schultz Sep 10, 2019
9d61e84
use pytest assertions
patrick-schultz Sep 11, 2019
eadff26
address comments
patrick-schultz Sep 11, 2019
cb8fd48
fix
patrick-schultz Sep 13, 2019
9f4c298
allow cse on ir with free variables
patrick-schultz Sep 13, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions hail/python/hail/backend/backend.py
Expand Up @@ -6,7 +6,7 @@
from hail.expr.table_type import *
from hail.expr.matrix_type import *
from hail.expr.blockmatrix_type import *
from hail.ir.renderer import Renderer
from hail.ir.renderer import CSERenderer, Renderer
from hail.table import Table
from hail.matrixtable import MatrixTable

Expand Down Expand Up @@ -99,7 +99,7 @@ def fs(self):

def _to_java_ir(self, ir):
if not hasattr(ir, '_jir'):
r = Renderer(stop_at_jir=True)
r = CSERenderer(stop_at_jir=True)
# FIXME parse should be static
ir._jir = ir.parse(r(ir), ir_map=r.jirs)
return ir._jir
Expand Down
14 changes: 13 additions & 1 deletion hail/python/hail/ir/base_ir.py
Expand Up @@ -33,7 +33,10 @@ def render_head(self, r):
head_str = self.head_str()
if head_str != '':
head_str = f' {head_str}'
return f'({self._ir_name()}{head_str}'
trailing_space = ''
if len(self.children) > 0:
trailing_space = ' '
return f'({self._ir_name()}{head_str}{trailing_space}'

def render_tail(self, r):
return ')'
Expand Down Expand Up @@ -89,6 +92,9 @@ def __hash__(self):
def new_block(self, i: int) -> bool:
...

def renderable_new_block(self, i: int) -> bool:
return self.new_block(i)

@staticmethod
def is_effectful() -> bool:
return False
Expand All @@ -113,9 +119,15 @@ def scan_bindings(self, i: int, default_value=None):
def uses_agg_context(self, i: int) -> bool:
return False

def renderable_uses_agg_context(self, i: int) -> bool:
return self.uses_agg_context(i)

def uses_scan_context(self, i: int) -> bool:
return False

def renderable_uses_scan_context(self, i: int) -> bool:
return self.uses_scan_context(i)

def child_context_without_bindings(self, i: int, parent_context):
(eval_c, agg_c, scan_c) = parent_context
if self.uses_agg_context(i):
Expand Down
9 changes: 9 additions & 0 deletions hail/python/hail/ir/ir.py
Expand Up @@ -1409,6 +1409,9 @@ def new_block(self, i: int) -> bool:
n = len(self.constructor_args)
return self.init_op_args and i < n + len(self.init_op_args)

def renderable_new_block(self, i: int) -> bool:
return i <= 1


class ApplyAggOp(BaseApplyAggOp):
@typecheck_method(agg_op=str,
Expand All @@ -1421,6 +1424,9 @@ def __init__(self, agg_op, constructor_args, init_op_args, seq_op_args):
def uses_agg_context(self, i: int):
return i >= len(self.constructor_args) + (len(self.init_op_args) if self.init_op_args else 0)

def renderable_uses_agg_context(self, i: int):
return i == 2


class ApplyScanOp(BaseApplyAggOp):
@typecheck_method(agg_op=str,
Expand All @@ -1433,6 +1439,9 @@ def __init__(self, agg_op, constructor_args, init_op_args, seq_op_args):
def uses_scan_context(self, i: int):
return i >= len(self.constructor_args) + (len(self.init_op_args) if self.init_op_args else 0)

def renderable_uses_agg_context(self, i: int):
return i == 2


class Begin(IR):
@typecheck_method(xs=sequenceof(IR))
Expand Down