diff --git a/buildscripts/hpat-conda-recipe/meta.yaml b/buildscripts/hpat-conda-recipe/meta.yaml index a3877c641..9e148262f 100644 --- a/buildscripts/hpat-conda-recipe/meta.yaml +++ b/buildscripts/hpat-conda-recipe/meta.yaml @@ -23,7 +23,7 @@ requirements: host: - python - setuptools - - numba ==0.45 + - numba ==0.46 - numpy - pandas >=0.23 - pyarrow ==0.14.1 @@ -44,7 +44,7 @@ requirements: - pyarrow ==0.14.1 - arrow-cpp ==0.14.1 - boost - - numba ==0.45 + - numba ==0.46 - mpich # [not win] - impi_rt # [win] - conda-package-handling ==1.3.11 #[win] diff --git a/hpat/compiler.py b/hpat/compiler.py index 501b329ba..70cec1643 100644 --- a/hpat/compiler.py +++ b/hpat/compiler.py @@ -4,116 +4,29 @@ import hpat.hiframes import hpat.hiframes.hiframes_untyped import hpat.hiframes.hiframes_typed -from hpat.hiframes.hiframes_untyped import HiFrames -from hpat.hiframes.hiframes_typed import HiFramesTyped +from hpat.hiframes.hiframes_untyped import HiFramesPass +from hpat.hiframes.hiframes_typed import HiFramesTypedPass from hpat.hiframes.dataframe_pass import DataFramePass import numba import numba.compiler +from numba.compiler import DefaultPassBuilder from numba import ir_utils, ir, postproc from numba.targets.registry import CPUDispatcher from numba.ir_utils import guard, get_definition from numba.inline_closurecall import inline_closure_call, InlineClosureCallPass +from numba.typed_passes import (NopythonTypeInference, AnnotateTypes, ParforPass, IRLegalization) +from numba.untyped_passes import (DeadBranchPrune, InlineInlinables, InlineClosureLikes) from hpat import config +from hpat.distributed import DistributedPass import hpat.io if config._has_h5py: from hpat.io import pio +from numba.compiler_machinery import FunctionPass, register_pass + # workaround for Numba #3876 issue with large labels in mortgage benchmark binding.set_option("tmp", "-non-global-value-max-name-size=2048") -# this is for previous version of pipeline manipulation (numba hpat_req <0.38) -# def stage_io_pass(pipeline): -# """ -# Convert IO calls -# """ -# # Ensure we have an IR and type information. -# assert pipeline.func_ir -# if config._has_h5py: -# io_pass = pio.PIO(pipeline.func_ir, pipeline.locals) -# io_pass.run() -# -# -# def stage_distributed_pass(pipeline): -# """ -# parallelize for distributed-memory -# """ -# # Ensure we have an IR and type information. -# assert pipeline.func_ir -# dist_pass = DistributedPass(pipeline.func_ir, pipeline.typingctx, -# pipeline.type_annotation.typemap, pipeline.type_annotation.calltypes) -# dist_pass.run() -# -# -# def stage_df_pass(pipeline): -# """ -# Convert DataFrame calls -# """ -# # Ensure we have an IR and type information. -# assert pipeline.func_ir -# df_pass = HiFrames(pipeline.func_ir, pipeline.typingctx, -# pipeline.args, pipeline.locals) -# df_pass.run() -# -# -# def stage_df_typed_pass(pipeline): -# """ -# Convert HiFrames after typing -# """ -# # Ensure we have an IR and type information. -# assert pipeline.func_ir -# df_pass = HiFramesTyped(pipeline.func_ir, pipeline.typingctx, -# pipeline.type_annotation.typemap, pipeline.type_annotation.calltypes) -# df_pass.run() -# -# -# def stage_inline_pass(pipeline): -# """ -# Inline function calls (to enable distributed pass analysis) -# """ -# # Ensure we have an IR and type information. -# assert pipeline.func_ir -# inline_calls(pipeline.func_ir) -# -# -# def stage_repeat_inline_closure(pipeline): -# assert pipeline.func_ir -# inline_pass = InlineClosureCallPass( -# pipeline.func_ir, pipeline.flags.auto_parallel) -# inline_pass.run() -# post_proc = postproc.PostProcessor(pipeline.func_ir) -# post_proc.run() -# -# -# def add_hpat_stages(pipeline_manager, pipeline): -# pp = pipeline_manager.pipeline_stages['nopython'] -# new_pp = [] -# for (func, desc) in pp: -# if desc == 'nopython frontend': -# # before type inference: add inline calls pass, -# # untyped hiframes pass, hdf5 io -# # also repeat inline closure pass to inline df stencils -# new_pp.append( -# (lambda: stage_inline_pass(pipeline), "inline funcs")) -# new_pp.append((lambda: stage_df_pass( -# pipeline), "convert DataFrames")) -# new_pp.append((lambda: stage_io_pass( -# pipeline), "replace IO calls")) -# new_pp.append((lambda: stage_repeat_inline_closure( -# pipeline), "repeat inline closure")) -# # need to handle string array exprs before nopython rewrites converts -# # them to arrayexpr. -# # since generic_rewrites has the same description, we check func name -# if desc == 'nopython rewrites' and 'generic_rewrites' not in str(func): -# new_pp.append((lambda: stage_df_typed_pass( -# pipeline), "typed hiframes pass")) -# if desc == 'nopython mode backend': -# # distributed pass after parfor pass and before lowering -# new_pp.append((lambda: stage_distributed_pass( -# pipeline), "convert to distributed")) -# new_pp.append((func, desc)) -# pipeline_manager.pipeline_stages['nopython'] = new_pp - - def inline_calls(func_ir, _locals): work_list = list(func_ir.blocks.items()) while work_list: @@ -150,145 +63,104 @@ def inline_calls(func_ir, _locals): # CFG simplification fixes this case func_ir.blocks = ir_utils.simplify_CFG(func_ir.blocks) +#TODO: remove these helper functions when Numba provide appropriate way to manipulate passes +def pass_position(pm, location): + assert pm.passes + pm._validate_pass(location) + for idx, (x, _) in enumerate(pm.passes): + if x == location: + return idx + + raise ValueError("Could not find pass %s" % location) + + +def add_pass_before(pm, pass_cls, location): + assert pm.passes + pm._validate_pass(pass_cls) + position = pass_position(pm, location) + pm.passes.insert(position, (pass_cls, str(pass_cls))) + + # if a pass has been added, it's not finalized + pm._finalized = False + +def replace_pass(pm, pass_cls, location): + assert pm.passes + pm._validate_pass(pass_cls) + position = pass_position(pm, location) + pm[position] = (pass_cls, str(pass_cls)) + + # if a pass has been added, it's not finalized + pm._finalized = False + +@register_pass(mutates_CFG=True, analysis_only=False) +class InlinePass(FunctionPass): + _name = "hpat_inline_pass" + + def __init__(self): + pass -class HPATPipeline(numba.compiler.BasePipeline): + def run_pass(self, state): + inline_calls(state.func_ir, state.locals) + return True + +@register_pass(mutates_CFG=True, analysis_only=False) +class PostprocessorPass(FunctionPass): + _name = "hpat_postprocessor_pass" + + def __init__(self): + pass + + def run_pass(self, state): + post_proc = postproc.PostProcessor(state.func_ir) + post_proc.run() + return True + +class HPATPipeline(numba.compiler.CompilerBase): """HPAT compiler pipeline """ - def define_pipelines(self, pm): + def define_pipelines(self): name = 'hpat' - pm.create_pipeline(name) - self.add_preprocessing_stage(pm) - self.add_with_handling_stage(pm) - self.add_pre_typing_stage(pm) - pm.add_stage(self.stage_inline_pass, "inline funcs") - pm.add_stage(self.stage_df_pass, "convert DataFrames") - # pm.add_stage(self.stage_io_pass, "replace IO calls") - # repeat inline closure pass to inline df stencils - pm.add_stage(self.stage_repeat_inline_closure, "repeat inline closure") - self.add_typing_stage(pm) - # breakup optimization stage since df_typed needs to run before - # rewrites - # e.g. need to handle string array exprs before nopython rewrites - # converts them to arrayexpr. - # self.add_optimization_stage(pm) - # hiframes typed pass should be before pre_parfor since variable types - # need updating, and A.call to np.call transformation is invalid for - # Series (e.g. S.var is not the same as np.var(S)) - pm.add_stage(self.stage_dataframe_pass, "typed dataframe pass") - pm.add_stage(self.stage_df_typed_pass, "typed hiframes pass") - pm.add_stage(self.stage_pre_parfor_pass, "Preprocessing for parfors") - if not self.flags.no_rewrites: - pm.add_stage(self.stage_nopython_rewrites, "nopython rewrites") - if self.flags.auto_parallel.enabled: - pm.add_stage(self.stage_parfor_pass, "convert to parfors") - pm.add_stage(self.stage_distributed_pass, "convert to distributed") - pm.add_stage(self.stage_ir_legalization, - "ensure IR is legal prior to lowering") - self.add_lowering_stage(pm) - self.add_cleanup_stage(pm) + pm = DefaultPassBuilder.define_nopython_pipeline(self.state) - def stage_inline_pass(self): - """ - Inline function calls (to enable distributed pass analysis) - """ - # Ensure we have an IR and type information. - assert self.func_ir - inline_calls(self.func_ir, self.locals) + add_pass_before(pm, InlinePass, InlineClosureLikes) + pm.add_pass_after(HiFramesPass, InlinePass) + pm.add_pass_after(DataFramePass, AnnotateTypes) + pm.add_pass_after(PostprocessorPass, AnnotateTypes) + pm.add_pass_after(HiFramesTypedPass, DataFramePass) + pm.add_pass_after(DistributedPass, ParforPass) + pm.finalize() - def stage_df_pass(self): - """ - Convert DataFrame calls - """ - # Ensure we have an IR and type information. - assert self.func_ir - df_pass = HiFrames(self.func_ir, self.typingctx, - self.args, self.locals, self.metadata) - df_pass.run() + return [pm] - def stage_io_pass(self): - """ - Convert IO calls - """ - # Ensure we have an IR and type information. - assert self.func_ir - if config._has_h5py: - io_pass = pio.PIO(self.func_ir, self.locals) - io_pass.run() - def stage_repeat_inline_closure(self): - assert self.func_ir - inline_pass = InlineClosureCallPass( - self.func_ir, self.flags.auto_parallel, typed=True) - inline_pass.run() - post_proc = postproc.PostProcessor(self.func_ir) - post_proc.run() - - def stage_distributed_pass(self): - """ - parallelize for distributed-memory - """ - # Ensure we have an IR and type information. - assert self.func_ir - from hpat.distributed import DistributedPass - dist_pass = DistributedPass( - self.func_ir, self.typingctx, self.targetctx, - self.type_annotation.typemap, self.type_annotation.calltypes, - self.metadata) - dist_pass.run() +@register_pass(mutates_CFG=True, analysis_only=False) +class ParforSeqPass(FunctionPass): + _name = "hpat_parfor_seq_pass" - def stage_df_typed_pass(self): - """ - Convert HiFrames after typing - """ - # Ensure we have an IR and type information. - assert self.func_ir - df_pass = HiFramesTyped(self.func_ir, self.typingctx, - self.type_annotation.typemap, - self.type_annotation.calltypes) - df_pass.run() + def __init__(self): + pass - def stage_dataframe_pass(self): - """ - Convert DataFrames after typing - """ - # Ensure we have an IR and type information. - assert self.func_ir - df_pass = DataFramePass(self.func_ir, self.typingctx, - self.type_annotation.typemap, - self.type_annotation.calltypes) - df_pass.run() + def run_pass(self, state): + numba.parfor.lower_parfor_sequential( + state.typingctx, state.func_ir, state.typemap, state.calltypes) + return True class HPATPipelineSeq(HPATPipeline): """HPAT pipeline without the distributed pass (used in rolling kernels) """ - def define_pipelines(self, pm): + def define_pipelines(self): name = 'hpat_seq' - pm.create_pipeline(name) - self.add_preprocessing_stage(pm) - self.add_with_handling_stage(pm) - self.add_pre_typing_stage(pm) - pm.add_stage(self.stage_inline_pass, "inline funcs") - pm.add_stage(self.stage_df_pass, "convert DataFrames") - pm.add_stage(self.stage_repeat_inline_closure, "repeat inline closure") - self.add_typing_stage(pm) - # TODO: dataframe pass needed? - pm.add_stage(self.stage_dataframe_pass, "typed dataframe pass") - pm.add_stage(self.stage_df_typed_pass, "typed hiframes pass") - pm.add_stage(self.stage_pre_parfor_pass, "Preprocessing for parfors") - if not self.flags.no_rewrites: - pm.add_stage(self.stage_nopython_rewrites, "nopython rewrites") - if self.flags.auto_parallel.enabled: - pm.add_stage(self.stage_parfor_pass, "convert to parfors") - # pm.add_stage(self.stage_distributed_pass, "convert to distributed") - pm.add_stage(self.stage_lower_parfor_seq, "parfor seq lower") - pm.add_stage(self.stage_ir_legalization, - "ensure IR is legal prior to lowering") - self.add_lowering_stage(pm) - self.add_cleanup_stage(pm) + pm = DefaultPassBuilder.define_nopython_pipeline(self.state) - def stage_lower_parfor_seq(self): - numba.parfor.lower_parfor_sequential( - self.typingctx, self.func_ir, self.typemap, self.calltypes) + add_pass_before(pm, InlinePass, InlineClosureLikes) + pm.add_pass_after(HiFramesPass, InlinePass) + pm.add_pass_after(DataFramePass, AnnotateTypes) + pm.add_pass_after(PostprocessorPass, AnnotateTypes) + pm.add_pass_after(HiFramesTypedPass, DataFramePass) + add_pass_before(pm, ParforSeqPass, IRLegalization) + pm.finalize() + + return [pm] \ No newline at end of file diff --git a/hpat/decorators.py b/hpat/decorators.py index 9e030fd85..49948eaf3 100644 --- a/hpat/decorators.py +++ b/hpat/decorators.py @@ -90,7 +90,4 @@ def jit(signature_or_function=None, **options): else: hpat.config.config_transport_mpi = False - # this is for previous version of pipeline manipulation (numba hpat_req <0.38) - # from .compiler import add_hpat_stages - # return numba.jit(signature_or_function, user_pipeline_funcs=[add_hpat_stages], **options) return numba.jit(signature_or_function, pipeline_class=hpat.compiler.HPATPipeline, **options) diff --git a/hpat/distributed.py b/hpat/distributed.py index 2bbdddbbf..5a75a6a6c 100644 --- a/hpat/distributed.py +++ b/hpat/distributed.py @@ -47,6 +47,8 @@ wrap_parfor_blocks, unwrap_parfor_blocks) +from numba.compiler_machinery import FunctionPass, register_pass + import hpat import hpat.utils from hpat import distributed_api, distributed_lower @@ -67,66 +69,38 @@ ReplaceFunc, gen_getitem, is_call, - is_const_slice) + is_const_slice, + update_globals) from hpat.hiframes.pd_dataframe_ext import DataFrameType - distributed_run_extensions = {} # analysis data for debugging dist_analysis = None fir_text = None - -class DistributedPass(object): +@register_pass(mutates_CFG=True, analysis_only=False) +class DistributedPass(FunctionPass): """The summary of the class should be here for example below is the summary line for this class - This class analyzes program and transforms to distributed + This is an adapter for a new numba passes interface. Numba pass must be stateless. This class wraps statefull DistributedPassImpl """ - def __init__(self, func_ir, typingctx, targetctx, typemap, calltypes, - metadata): - """ The summary for the class or init method can be mentioned here - - The details can be multiple lines. You can mention all the supported - and not supported args here or you can do it in ``Parameters`` section - - .. only:: developer - - The content for **Developer's Guide** should go here - - .. only:: user + _name = "distributed_pass" + def __init__(self): + pass - The content here is visible only to the **User's Guide** So all user specific content should be added here + def run_pass(self, state): + return DistributedPassImpl(state).run_pass() - Note - ---- - Do not include the `self` paramter in the ``Parameters`` section - Parameters - ----------- - func_ir : str - description of `func_ir`.(optional) you can mention supported or not here - typingctx : TypeForThisParam - description of `typingctx` - targetctx : TypeForThisParam - description of `targetctx` - typemap : TypeForThisParam - description of `typemap` - calltypes : TypeForThisParam - description of `calltypes` - metadata : TypeForThisParam - description of `metadata` - - """ +class DistributedPassImpl(object): + """The summary of the class should be here for example below is the summary line for this class - self.func_ir = func_ir - self.typingctx = typingctx - self.targetctx = targetctx - self.typemap = typemap - self.calltypes = calltypes - self.metadata = metadata + This class analyzes program and transforms to distributed + """ + def __init__(self, state): self._dist_analysis = None self._T_arrs = None # set of transposed arrays (taken from analysis) @@ -147,15 +121,17 @@ def __init__(self, func_ir, typingctx, targetctx, typemap, calltypes, # size for 1DVar allocs and parfors self.oneDVar_len_vars = {} - def run(self): - remove_dels(self.func_ir.blocks) - dprint_func_ir(self.func_ir, "starting distributed pass") - self.func_ir._definitions = build_definitions(self.func_ir.blocks) + self.state = state + + def run_pass(self): + remove_dels(self.state.func_ir.blocks) + dprint_func_ir(self.state.func_ir, "starting distributed pass") + self.state.func_ir._definitions = build_definitions(self.state.func_ir.blocks) dist_analysis_pass = DistributedAnalysis( - self.func_ir, self.typemap, self.calltypes, self.typingctx, - self.metadata) + self.state.func_ir, self.state.typemap, self.state.calltypes, self.state.typingctx, + self.state.metadata) self._dist_analysis = dist_analysis_pass.run() - # dprint_func_ir(self.func_ir, "after analysis distributed") + # dprint_func_ir(self.state.func_ir, "after analysis distributed") self._T_arrs = dist_analysis_pass._T_arrs self._parallel_accesses = dist_analysis_pass._parallel_accesses @@ -163,20 +139,20 @@ def run(self): print("distributions: ", self._dist_analysis) self._gen_dist_inits() - self.func_ir._definitions = build_definitions(self.func_ir.blocks) - self.func_ir.blocks = self._run_dist_pass(self.func_ir.blocks) - self.func_ir.blocks = self._dist_prints(self.func_ir.blocks) - remove_dead(self.func_ir.blocks, self.func_ir.arg_names, self.func_ir, self.typemap) - dprint_func_ir(self.func_ir, "after distributed pass") + self.state.func_ir._definitions = build_definitions(self.state.func_ir.blocks) + self.state.func_ir.blocks = self._run_dist_pass(self.state.func_ir.blocks) + self.state.func_ir.blocks = self._dist_prints(self.state.func_ir.blocks) + remove_dead(self.state.func_ir.blocks, self.state.func_ir.arg_names, self.state.func_ir, self.state.typemap) + dprint_func_ir(self.state.func_ir, "after distributed pass") lower_parfor_sequential( - self.typingctx, self.func_ir, self.typemap, self.calltypes) + self.state.typingctx, self.state.func_ir, self.state.typemap, self.state.calltypes) if hpat.multithread_mode: # parfor params need to be updated for multithread_mode since some # new variables like alloc_start are introduced by distributed pass # and are used in later parfors parfor_ids = get_parfor_params( - self.func_ir.blocks, True, defaultdict(list)) - post_proc = postproc.PostProcessor(self.func_ir) + self.state.func_ir.blocks, True, defaultdict(list)) + post_proc = postproc.PostProcessor(self.state.func_ir) post_proc.run() # save data for debug and test @@ -184,10 +160,12 @@ def run(self): dist_analysis = self._dist_analysis import io str_io = io.StringIO() - self.func_ir.dump(str_io) + self.state.func_ir.dump(str_io) fir_text = str_io.getvalue() str_io.close() + return True + def _run_dist_pass(self, blocks): """This function does something""" topo_order = find_topo_order(blocks) @@ -202,13 +180,13 @@ def _run_dist_pass(self, blocks): if type(inst) in distributed_run_extensions: f = distributed_run_extensions[type(inst)] out_nodes = f(inst, self._dist_analysis.array_dists, - self.typemap, self.calltypes, self.typingctx, - self.targetctx, self) + self.state.typemap, self.state.calltypes, self.state.typingctx, + self.state.targetctx, self) elif isinstance(inst, Parfor): out_nodes = self._run_parfor(inst, namevar_table) # run dist pass recursively p_blocks = wrap_parfor_blocks(inst) - # build_definitions(p_blocks, self.func_ir._definitions) + # build_definitions(p_blocks, self.state.func_ir._definitions) self._run_dist_pass(p_blocks) unwrap_parfor_blocks(inst) elif isinstance(inst, ir.Assign): @@ -217,7 +195,7 @@ def _run_dist_pass(self, blocks): if isinstance(rhs, ir.Expr): out_nodes = self._run_expr(inst, namevar_table) elif isinstance(rhs, ir.Var) and (self._is_1D_arr(rhs.name) - and not is_array_container(self.typemap, rhs.name)): + and not is_array_container(self.state.typemap, rhs.name)): self._array_starts[lhs] = self._array_starts[rhs.name] self._array_counts[lhs] = self._array_counts[rhs.name] self._array_sizes[lhs] = self._array_sizes[rhs.name] @@ -259,10 +237,11 @@ def _run_dist_pass(self, blocks): block.body = new_body + block.body[i:] # TODO: use Parfor loop blocks when replacing funcs in # parfor loop body - inline_closure_call(self.func_ir, rp_func.glbls, - block, len(new_body), rp_func.func, self.typingctx, + update_globals(rp_func.func, rp_func.glbls) + inline_closure_call(self.state.func_ir, rp_func.glbls, + block, len(new_body), rp_func.func, self.state.typingctx, rp_func.arg_types, - self.typemap, self.calltypes, work_list) + self.state.typemap, self.state.calltypes, work_list) replaced = True break else: @@ -295,7 +274,7 @@ def _run_expr(self, inst, namevar_table): if (rhs.op == 'static_getitem' and rhs.value.name in self._shape_attrs): arr = self._shape_attrs[rhs.value.name] - ndims = self.typemap[arr].ndim + ndims = self.state.typemap[arr].ndim if arr not in self._T_arrs and rhs.index == 0: # return parallel size if self._is_1D_arr(arr): @@ -358,9 +337,9 @@ def _gen_1D_Var_len(self, arr): def f(A, op): # pragma: no cover c = len(A) res = hpat.distributed_api.dist_reduce(c, op) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[arr.name], types.int32), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[arr.name], types.int32), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes( f_block, [arr, ir.Const(Reduce_Type.Sum.value, arr.loc)]) nodes = f_block.body[:-3] # remove none return @@ -368,38 +347,38 @@ def f(A, op): # pragma: no cover def _gen_dist_inits(self): # add initializations - topo_order = find_topo_order(self.func_ir.blocks) - first_block = self.func_ir.blocks[topo_order[0]] + topo_order = find_topo_order(self.state.func_ir.blocks) + first_block = self.state.func_ir.blocks[topo_order[0]] # set scope and loc of generated code to the first variable in block scope = first_block.scope loc = first_block.loc out = [] self._set1_var = ir.Var(scope, mk_unique_var("$const_parallel"), loc) - self.typemap[self._set1_var.name] = types.int64 + self.state.typemap[self._set1_var.name] = types.int64 set1_assign = ir.Assign(ir.Const(1, loc), self._set1_var, loc) out.append(set1_assign) self._set0_var = ir.Var(scope, mk_unique_var("$const_parallel"), loc) - self.typemap[self._set0_var.name] = types.int64 + self.state.typemap[self._set0_var.name] = types.int64 set0_assign = ir.Assign(ir.Const(0, loc), self._set0_var, loc) out.append(set0_assign) # g_dist_var = Global(hpat.distributed_api) g_dist_var = ir.Var(scope, mk_unique_var("$distributed_g_var"), loc) self._g_dist_var = g_dist_var - self.typemap[g_dist_var.name] = types.misc.Module(hpat.distributed_api) + self.state.typemap[g_dist_var.name] = types.misc.Module(hpat.distributed_api) g_dist = ir.Global('distributed_api', hpat.distributed_api, loc) g_dist_assign = ir.Assign(g_dist, g_dist_var, loc) # attr call: rank_attr = getattr(g_dist_var, get_rank) rank_attr_call = ir.Expr.getattr(g_dist_var, "get_rank", loc) rank_attr_var = ir.Var(scope, mk_unique_var("$get_rank_attr"), loc) - self.typemap[rank_attr_var.name] = get_global_func_typ( + self.state.typemap[rank_attr_var.name] = get_global_func_typ( distributed_api.get_rank) rank_attr_assign = ir.Assign(rank_attr_call, rank_attr_var, loc) # rank_var = hpat.distributed_api.get_rank() rank_var = ir.Var(scope, mk_unique_var("$rank"), loc) - self.typemap[rank_var.name] = types.int32 + self.state.typemap[rank_var.name] = types.int32 rank_call = ir.Expr.call(rank_attr_var, [], (), loc) - self.calltypes[rank_call] = self.typemap[rank_attr_var.name].get_call_type( - self.typingctx, [], {}) + self.state.calltypes[rank_call] = self.state.typemap[rank_attr_var.name].get_call_type( + self.state.typingctx, [], {}) rank_assign = ir.Assign(rank_call, rank_var, loc) self._rank_var = rank_var out += [g_dist_assign, rank_attr_assign, rank_assign] @@ -407,15 +386,15 @@ def _gen_dist_inits(self): # attr call: size_attr = getattr(g_dist_var, get_size) size_attr_call = ir.Expr.getattr(g_dist_var, "get_size", loc) size_attr_var = ir.Var(scope, mk_unique_var("$get_size_attr"), loc) - self.typemap[size_attr_var.name] = get_global_func_typ( + self.state.typemap[size_attr_var.name] = get_global_func_typ( distributed_api.get_size) size_attr_assign = ir.Assign(size_attr_call, size_attr_var, loc) # size_var = hpat.distributed_api.get_size() size_var = ir.Var(scope, mk_unique_var("$dist_size"), loc) - self.typemap[size_var.name] = types.int32 + self.state.typemap[size_var.name] = types.int32 size_call = ir.Expr.call(size_attr_var, [], (), loc) - self.calltypes[size_call] = self.typemap[size_attr_var.name].get_call_type( - self.typingctx, [], {}) + self.state.calltypes[size_call] = self.state.typemap[size_attr_var.name].get_call_type( + self.state.typingctx, [], {}) size_assign = ir.Assign(size_call, size_var, loc) self._size_var = size_var out += [size_attr_assign, size_assign] @@ -431,7 +410,7 @@ def _run_call(self, assign): func_name = "" func_mod = "" - fdef = guard(find_callname, self.func_ir, rhs, self.typemap) + fdef = guard(find_callname, self.state.func_ir, rhs, self.state.typemap) if fdef is None: # FIXME: since parfors are transformed and then processed # recursively, some funcs don't have definitions. The generated @@ -471,23 +450,23 @@ def _run_call(self, assign): return self._run_call_np(lhs, func_name, assign, rhs.args) # array.func calls - if isinstance(func_mod, ir.Var) and is_np_array(self.typemap, func_mod.name): + if isinstance(func_mod, ir.Var) and is_np_array(self.state.typemap, func_mod.name): return self._run_call_array(lhs, func_mod, func_name, assign, rhs.args) # df.func calls - if isinstance(func_mod, ir.Var) and isinstance(self.typemap[func_mod.name], DataFrameType): + if isinstance(func_mod, ir.Var) and isinstance(self.state.typemap[func_mod.name], DataFrameType): return self._run_call_df(lhs, func_mod, func_name, assign, rhs.args) # string_array.func_calls if (self._is_1D_arr(lhs) and isinstance(func_mod, ir.Var) - and self.typemap[func_mod.name] == string_array_type): + and self.state.typemap[func_mod.name] == string_array_type): if func_name == 'copy': self._array_starts[lhs] = self._array_starts[func_mod.name] self._array_counts[lhs] = self._array_counts[func_mod.name] self._array_sizes[lhs] = self._array_sizes[func_mod.name] if fdef == ('permutation', 'numpy.random'): - if self.typemap[rhs.args[0].name] == types.int64: + if self.state.typemap[rhs.args[0].name] == types.int64: self._array_sizes[lhs] = [rhs.args[0]] return self._run_permutation_int(assign, rhs.args) @@ -510,14 +489,14 @@ def _run_call(self, assign): arr = rhs.args[5].name ndims = len(self._array_starts[arr]) starts_var = ir.Var(scope, mk_unique_var("$h5_starts"), loc) - self.typemap[starts_var.name] = types.UniTuple( + self.state.typemap[starts_var.name] = types.UniTuple( types.int64, ndims) start_tuple_call = ir.Expr.build_tuple( self._array_starts[arr], loc) starts_assign = ir.Assign(start_tuple_call, starts_var, loc) rhs.args[2] = starts_var counts_var = ir.Var(scope, mk_unique_var("$h5_counts"), loc) - self.typemap[counts_var.name] = types.UniTuple( + self.state.typemap[counts_var.name] = types.UniTuple( types.int64, ndims) count_tuple_call = ir.Expr.build_tuple( self._array_counts[arr], loc) @@ -562,7 +541,7 @@ def f(fname, cindex, arr, out_dtype, start, count): # pragma: no cover and self._is_1D_arr(lhs)): arr = lhs size_var = rhs.args[2] - assert self.typemap[size_var.name] == types.intp + assert self.state.typemap[size_var.name] == types.intp self._array_sizes[arr] = [size_var] out, start_var, count_var = self._gen_1D_div(size_var, scope, loc, "$alloc", "get_node_portion", distributed_api.get_node_portion) @@ -575,10 +554,10 @@ def f(fname, cindex, start, count): # pragma: no cover return hpat.io.parquet_pio.read_parquet_str_parallel(fname, cindex, start, count) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[rhs.args[0].name], types.intp, + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[rhs.args[0].name], types.intp, types.intp, types.intp), - self.typemap, self.calltypes).blocks.popitem()[1] + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, rhs.args) out += f_block.body[:-2] out[-1].target = assign.target @@ -602,7 +581,7 @@ def f(connect_tp, dset_tp, col_id_tp, column_tp, schema_arr_tp, start, count): and self._is_1D_arr(lhs)): arr = lhs size_var = rhs.args[3] - assert self.typemap[size_var.name] == types.intp + assert self.state.typemap[size_var.name] == types.intp self._array_sizes[arr] = [size_var] out, start_var, count_var = self._gen_1D_div(size_var, scope, loc, "$alloc", "get_node_portion", distributed_api.get_node_portion) @@ -618,15 +597,15 @@ def f(connect_tp, dset_tp, col_id_tp, schema_arr_tp, start_tp, count_tp): # pra f_block = compile_to_numba_ir(f, {'hpat': hpat}, - self.typingctx, + self.state.typingctx, (hpat.io.xenon_ext.xe_connect_type, hpat.io.xenon_ext.xe_dset_type, types.intp, - self.typemap[rhs.args[3].name], + self.state.typemap[rhs.args[3].name], types.intp, types.intp), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, rhs.args) out += f_block.body[:-2] out[-1].target = assign.target @@ -729,7 +708,7 @@ def f(arr, bag, start, count): # pragma: no cover self._array_sizes[lhs] = self._array_sizes[in_arr] # set parallel flag to true true_var = ir.Var(scope, mk_unique_var("true_var"), loc) - self.typemap[true_var.name] = types.boolean + self.state.typemap[true_var.name] = types.boolean rhs.args[3] = true_var out = [ir.Assign(ir.Const(True, loc), true_var, loc), assign] @@ -743,7 +722,7 @@ def f(arr, bag, start, count): # pragma: no cover self._array_sizes[lhs] = self._array_sizes[in_arr] # set parallel flag to true true_var = ir.Var(scope, mk_unique_var("true_var"), loc) - self.typemap[true_var.name] = types.boolean + self.state.typemap[true_var.name] = types.boolean rhs.args[4] = true_var out = [ir.Assign(ir.Const(True, loc), true_var, loc), assign] @@ -758,7 +737,7 @@ def f(arr, bag, start, count): # pragma: no cover self._array_sizes[lhs] = self._array_sizes[in_arr] # set parallel flag to true true_var = ir.Var(scope, mk_unique_var("true_var"), loc) - self.typemap[true_var.name] = types.boolean + self.state.typemap[true_var.name] = types.boolean rhs.args[2] = true_var out = [ir.Assign(ir.Const(True, loc), true_var, loc), assign] @@ -824,8 +803,8 @@ def f(arr): if fdef == ('convert_rec_to_tup', 'hpat.hiframes.api'): # optimize Series back to back map pattern with tuples # TODO: create another optimization pass? - arg_def = guard(get_definition, self.func_ir, rhs.args[0]) - if (is_call(arg_def) and guard(find_callname, self.func_ir, arg_def) + arg_def = guard(get_definition, self.state.func_ir, rhs.args[0]) + if (is_call(arg_def) and guard(find_callname, self.state.func_ir, arg_def) == ('convert_tup_to_rec', 'hpat.hiframes.api')): assign.value = arg_def.args[0] return out @@ -845,7 +824,7 @@ def f(arr): arr = assign.target # gen len() using 1D_Var reduce approach. # TODO: refactor to avoid reduction for 1D - # arr_typ = self.typemap[arr.name] + # arr_typ = self.state.typemap[arr.name] ndim = 1 out += self._gen_1D_Var_len(arr) total_length = out[-1].target @@ -873,8 +852,8 @@ def f(arr): # output of mnb.predict is 1D with same size as 1st dimension of input # TODO: remove ml module and use new DAAL API if func_name == 'predict': - getattr_call = guard(get_definition, self.func_ir, func_var) - if (getattr_call and self.typemap[getattr_call.value.name] == hpat.ml.naive_bayes.mnb_type): + getattr_call = guard(get_definition, self.state.func_ir, func_var) + if (getattr_call and self.state.typemap[getattr_call.value.name] == hpat.ml.naive_bayes.mnb_type): in_arr = rhs.args[0].name self._array_starts[lhs] = [self._array_starts[in_arr][0]] self._array_counts[lhs] = [self._array_counts[in_arr][0]] @@ -912,7 +891,7 @@ def _run_call_np(self, lhs, func_name, assign, args): # args[1] = new_size_var # out.append(assign) - if (func_name == 'array' and is_array(self.typemap, args[0].name) and self._is_1D_arr(args[0].name)): + if (func_name == 'array' and is_array(self.state.typemap, args[0].name) and self._is_1D_arr(args[0].name)): in_arr = args[0].name self._array_starts[lhs] = self._array_starts[in_arr] self._array_counts[lhs] = self._array_counts[in_arr] @@ -922,7 +901,7 @@ def _run_call_np(self, lhs, func_name, assign, args): if (func_name in ['cumsum', 'cumprod', 'empty_like', 'zeros_like', 'ones_like', 'full_like', 'copy', 'ravel', 'ascontiguousarray'] and self._is_1D_arr(args[0].name)): if func_name == 'ravel': - assert self.typemap[args[0].name].ndim == 1, "only 1D ravel supported" + assert self.state.typemap[args[0].name].ndim == 1, "only 1D ravel supported" in_arr = args[0].name self._array_starts[lhs] = self._array_starts[in_arr] self._array_counts[lhs] = self._array_counts[in_arr] @@ -935,12 +914,12 @@ def _run_call_np(self, lhs, func_name, assign, args): # allocate output array # TODO: compute inplace if input array is dead out = mk_alloc( - self.typemap, - self.calltypes, + self.state.typemap, + self.state.calltypes, lhs_var, tuple( self._array_sizes[in_arr]), - self.typemap[in_arr].dtype, + self.state.typemap[in_arr].dtype, scope, loc) # generate distributed call @@ -948,19 +927,19 @@ def _run_call_np(self, lhs, func_name, assign, args): dist_func_name = "dist_" + func_name dist_func = getattr(distributed_api, dist_func_name) dist_attr_call = ir.Expr.getattr(self._g_dist_var, dist_func_name, loc) - self.typemap[dist_attr_var.name] = get_global_func_typ(dist_func) + self.state.typemap[dist_attr_var.name] = get_global_func_typ(dist_func) dist_func_assign = ir.Assign(dist_attr_call, dist_attr_var, loc) err_var = ir.Var(scope, mk_unique_var("$dist_err_var"), loc) - self.typemap[err_var.name] = types.int32 + self.state.typemap[err_var.name] = types.int32 dist_call = ir.Expr.call(dist_attr_var, [in_arr_var, lhs_var], (), loc) - self.calltypes[dist_call] = self.typemap[dist_attr_var.name].get_call_type( - self.typingctx, [self.typemap[in_arr], self.typemap[lhs]], {}) + self.state.calltypes[dist_call] = self.state.typemap[dist_attr_var.name].get_call_type( + self.state.typingctx, [self.state.typemap[in_arr], self.state.typemap[lhs]], {}) dist_assign = ir.Assign(dist_call, err_var, loc) return out + [dist_func_assign, dist_assign] # sum over the first axis is distributed, A.sum(0) if func_name == 'sum' and len(args) == 2: - axis_def = guard(get_definition, self.func_ir, args[1]) + axis_def = guard(get_definition, self.state.func_ir, args[1]) if isinstance(axis_def, ir.Const) and axis_def.value == 0: reduce_op = Reduce_Type.Sum reduce_var = assign.target @@ -971,7 +950,7 @@ def _run_call_np(self, lhs, func_name, assign, args): if func_name == 'stack' and self._is_1D_arr(lhs): # TODO: generalize - in_arrs, _ = guard(find_build_sequence, self.func_ir, args[0]) + in_arrs, _ = guard(find_build_sequence, self.state.func_ir, args[0]) arr0 = in_arrs[0].name self._array_starts[lhs] = [self._array_starts[arr0][0], None] self._array_counts[lhs] = [self._array_counts[arr0][0], None] @@ -989,7 +968,7 @@ def _run_call_array(self, lhs, arr, func_name, assign, args): # HACK support A.reshape(n, 1) for 1D_Var if func_name == 'reshape' and self._is_1D_Var_arr(arr.name): - assert len(args) == 2 and guard(find_const, self.func_ir, args[1]) == 1 + assert len(args) == 2 and guard(find_const, self.state.func_ir, args[1]) == 1 assert args[0].name in self.oneDVar_len_vars size_var = args[0] out, new_size_var = self._fix_1D_Var_alloc(size_var, lhs, assign.target.scope, assign.loc) @@ -1005,7 +984,7 @@ def _run_call_array(self, lhs, arr, func_name, assign, args): if func_name == 'transpose' and self._is_1D_arr(lhs): # Currently only 1D arrays are supported assert self._is_1D_arr(arr.name) - ndim = self.typemap[arr.name].ndim + ndim = self.state.typemap[arr.name].ndim self._array_starts[lhs] = [-1] * ndim self._array_counts[lhs] = [-1] * ndim self._array_sizes[lhs] = [-1] * ndim @@ -1051,17 +1030,17 @@ def _run_call_df(self, lhs, df, func_name, assign, args): # str_out = df2.to_csv(None, header=header) # hpat.io.np_io._file_write_parallel(fname, str_out) - df_typ = self.typemap[df.name] + df_typ = self.state.typemap[df.name] rhs = assign.value fname = args[0] # update df index and get to_csv from new df nodes = self._fix_parallel_df_index(df) new_df = nodes[-1].target - new_df_typ = self.typemap[new_df.name] + new_df_typ = self.state.typemap[new_df.name] new_to_csv = ir.Expr.getattr(new_df, 'to_csv', new_df.loc) new_func = ir.Var(new_df.scope, mk_unique_var('func'), new_df.loc) - self.typemap[new_func.name] = self.typingctx.resolve_getattr( + self.state.typemap[new_func.name] = self.state.typingctx.resolve_getattr( new_df_typ, 'to_csv') nodes.append(ir.Assign(new_to_csv, new_func, new_df.loc)) rhs.func = new_func @@ -1069,7 +1048,7 @@ def _run_call_df(self, lhs, df, func_name, assign, args): # # header = header and is_root kws = dict(rhs.kws) true_var = ir.Var(assign.target.scope, mk_unique_var('true'), rhs.loc) - self.typemap[true_var.name] = types.bool_ + self.state.typemap[true_var.name] = types.bool_ nodes.append( ir.Assign(ir.Const(True, new_df.loc), true_var, new_df.loc)) header_var = self._get_arg( @@ -1083,31 +1062,31 @@ def _run_call_df(self, lhs, df, func_name, assign, args): rhs.kws = kws # fix to_csv() type to have None as 1st arg - call_type = self.calltypes.pop(rhs) + call_type = self.state.calltypes.pop(rhs) arg_typs = list((types.none,) + call_type.args[1:]) arg_typs[5] = types.bool_ arg_typs = tuple(arg_typs) - # self.calltypes[rhs] = self.typemap[rhs.func.name].get_call_type( - # self.typingctx, arg_typs, {}) - self.calltypes[rhs] = numba.typing.Signature( + # self.state.calltypes[rhs] = self.state.typemap[rhs.func.name].get_call_type( + # self.state.typingctx, arg_typs, {}) + self.state.calltypes[rhs] = numba.typing.Signature( string_type, arg_typs, new_df_typ, call_type.pysig) # None as 1st arg none_var = ir.Var(assign.target.scope, mk_unique_var('none'), rhs.loc) - self.typemap[none_var.name] = types.none + self.state.typemap[none_var.name] = types.none none_assign = ir.Assign(ir.Const(None, rhs.loc), none_var, rhs.loc) nodes.append(none_assign) rhs.args[0] = none_var # str_out = df.to_csv(None) str_out = ir.Var(assign.target.scope, mk_unique_var('write_csv'), rhs.loc) - self.typemap[str_out.name] = string_type + self.state.typemap[str_out.name] = string_type new_assign = ir.Assign(rhs, str_out, rhs.loc) nodes.append(new_assign) # print_node = ir.Print([str_out], None, rhs.loc) - # self.calltypes[print_node] = signature(types.none, string_type) + # self.state.calltypes[print_node] = signature(types.none, string_type) # nodes.append(print_node) # TODO: fix lazy IO load @@ -1134,10 +1113,10 @@ def _gen_is_root_and_cond(self, cond_var): def f(cond): return cond & (hpat.distributed_api.get_rank() == 0) f_block = compile_to_numba_ir(f, {'hpat': hpat}, - self.typingctx, - (self.typemap[cond_var.name],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[cond_var.name],), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [cond_var]) nodes = f_block.body[:-2] return nodes @@ -1151,10 +1130,10 @@ def f(df): # pragma: no cover return df2 f_block = compile_to_numba_ir(f, {'hpat': hpat, 'np': np}, - self.typingctx, - (self.typemap[df.name],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[df.name],), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [df]) nodes = f_block.body[:-2] return nodes @@ -1167,26 +1146,26 @@ def f(lhs, n): hpat.distributed_lower.dist_permutation_int(lhs, n) f_block = compile_to_numba_ir(f, {'hpat': hpat}, - self.typingctx, - (self.typemap[lhs.name], types.intp), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[lhs.name], types.intp), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [lhs, n]) f_block.body = [assign] + f_block.body return f_block.body[:-3] # Returns an IR node that defines a variable holding the size of |dtype|. def dtype_size_assign_ir(self, dtype, scope, loc): - context = numba.targets.cpu.CPUContext(self.typingctx) + context = numba.targets.cpu.CPUContext(self.state.typingctx) dtype_size = context.get_abi_sizeof(context.get_data_type(dtype)) dtype_size_var = ir.Var(scope, mk_unique_var("dtype_size"), loc) - self.typemap[dtype_size_var.name] = types.intp + self.state.typemap[dtype_size_var.name] = types.intp return ir.Assign(ir.Const(dtype_size, loc), dtype_size_var, loc) def _run_permutation_array_index(self, lhs, rhs, idx): scope, loc = lhs.scope, lhs.loc - dtype = self.typemap[lhs.name].dtype - out = mk_alloc(self.typemap, self.calltypes, lhs, + dtype = self.state.typemap[lhs.name].dtype + out = mk_alloc(self.state.typemap, self.state.calltypes, lhs, (self._array_counts[lhs.name][0], *self._array_sizes[lhs.name][1:]), dtype, scope, loc) @@ -1195,15 +1174,15 @@ def f(lhs, lhs_len, dtype_size, rhs, idx, idx_len): lhs, lhs_len, dtype_size, rhs, idx, idx_len) f_block = compile_to_numba_ir(f, {'hpat': hpat}, - self.typingctx, - (self.typemap[lhs.name], + self.state.typingctx, + (self.state.typemap[lhs.name], types.intp, types.intp, - self.typemap[rhs.name], - self.typemap[idx.name], + self.state.typemap[rhs.name], + self.state.typemap[idx.name], types.intp), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] dtype_ir = self.dtype_size_assign_ir(dtype, scope, loc) out.append(dtype_ir) replace_arg_nodes(f_block, [lhs, self._array_sizes[lhs.name][0], @@ -1226,11 +1205,11 @@ def _run_reshape(self, assign, in_arr, args): out, new_local_shape_var = self._run_alloc(new_shape, lhs.name, scope, loc) # get actual tuple for mk_alloc if len(args) != 1: - sh_list = guard(find_build_tuple, self.func_ir, new_local_shape_var) + sh_list = guard(find_build_tuple, self.state.func_ir, new_local_shape_var) assert sh_list is not None, "invalid shape in reshape" new_local_shape_var = tuple(sh_list) - dtype = self.typemap[in_arr.name].dtype - out += mk_alloc(self.typemap, self.calltypes, lhs, + dtype = self.state.typemap[in_arr.name].dtype + out += mk_alloc(self.state.typemap, self.state.calltypes, lhs, new_local_shape_var, dtype, scope, loc) def f(lhs, in_arr, new_0dim_global_len, old_0dim_global_len, dtype_size): # pragma: no cover @@ -1238,10 +1217,10 @@ def f(lhs, in_arr, new_0dim_global_len, old_0dim_global_len, dtype_size): # pra lhs, in_arr, new_0dim_global_len, old_0dim_global_len, dtype_size) f_block = compile_to_numba_ir(f, {'hpat': hpat}, - self.typingctx, - (self.typemap[lhs.name], self.typemap[in_arr.name], + self.state.typingctx, + (self.state.typemap[lhs.name], self.state.typemap[in_arr.name], types.intp, types.intp, types.intp), - self.typemap, self.calltypes).blocks.popitem()[1] + self.state.typemap, self.state.calltypes).blocks.popitem()[1] dtype_ir = self.dtype_size_assign_ir(dtype, scope, loc) out.append(dtype_ir) replace_arg_nodes(f_block, [lhs, in_arr, self._array_sizes[lhs.name][0], @@ -1268,7 +1247,7 @@ def _run_call_rebalance_array(self, lhs, assign, args): return out arr = args[0] - ndim = self.typemap[arr.name].ndim + ndim = self.state.typemap[arr.name].ndim out = self._gen_1D_Var_len(arr) total_length = out[-1].target div_nodes, start_var, count_var = self._gen_1D_div( @@ -1287,9 +1266,9 @@ def _run_call_rebalance_array(self, lhs, assign, args): def f(arr, count): # pragma: no cover b_arr = hpat.distributed_api.rebalance_array_parallel(arr, count) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[arr.name], types.intp), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[arr.name], types.intp), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [arr, count_var]) out += f_block.body[:-3] out[-1].target = assign.target @@ -1299,8 +1278,8 @@ def _run_call_np_dot(self, lhs, assign, args): out = [assign] arg0 = args[0].name arg1 = args[1].name - ndim0 = self.typemap[arg0].ndim - ndim1 = self.typemap[arg1].ndim + ndim0 = self.state.typemap[arg0].ndim + ndim1 = self.state.typemap[arg1].ndim t0 = arg0 in self._T_arrs t1 = arg1 in self._T_arrs @@ -1317,7 +1296,7 @@ def _run_call_np_dot(self, lhs, assign, args): # special case were arg1 vector is treated as column vector # samples dot weights: np.dot(X,w) # output is 1D array same size as dim 0 of X - assert self.typemap[lhs].ndim == 1 + assert self.state.typemap[lhs].ndim == 1 assert self._is_1D_arr(lhs) self._array_starts[lhs] = [self._array_starts[arg0][0]] self._array_counts[lhs] = [self._array_counts[arg0][0]] @@ -1344,7 +1323,7 @@ def _run_alloc(self, size_var, lhs, scope, loc): new_size_var = None # size is single int var - if isinstance(size_var, ir.Var) and isinstance(self.typemap[size_var.name], types.Integer): + if isinstance(size_var, ir.Var) and isinstance(self.state.typemap[size_var.name], types.Integer): self._array_sizes[lhs] = [size_var] out, start_var, end_var = self._gen_1D_div(size_var, scope, loc, "$alloc", "get_node_portion", distributed_api.get_node_portion) @@ -1357,7 +1336,7 @@ def _run_alloc(self, size_var, lhs, scope, loc): if isinstance(size_var, ir.Var): # see if size_var is a 1D array's shape # it is already the local size, no need to transform - var_def = guard(get_definition, self.func_ir, size_var) + var_def = guard(get_definition, self.state.func_ir, size_var) oned_varnames = set(v for v in self._dist_analysis.array_dists if self._dist_analysis.array_dists[v] == Distribution.OneD) if (isinstance(var_def, ir.Expr) and var_def.op == 'getattr' @@ -1372,7 +1351,7 @@ def _run_alloc(self, size_var, lhs, scope, loc): #assert size_var.name in self._tuple_table # self._tuple_table[size_var.name] size_list = self._get_tuple_varlist(size_var, out) - size_list = [ir_utils.convert_size_to_var(s, self.typemap, scope, loc, out) + size_list = [ir_utils.convert_size_to_var(s, self.state.typemap, scope, loc, out) for s in size_list] # tuple of int vars else: @@ -1387,12 +1366,12 @@ def _run_alloc(self, size_var, lhs, scope, loc): new_size_list = copy.copy(size_list) new_size_list[0] = end_var tuple_var = ir.Var(scope, mk_unique_var("$tuple_var"), loc) - self.typemap[tuple_var.name] = types.containers.UniTuple( + self.state.typemap[tuple_var.name] = types.containers.UniTuple( types.intp, ndims) tuple_call = ir.Expr.build_tuple(new_size_list, loc) tuple_assign = ir.Assign(tuple_call, tuple_var, loc) out.append(tuple_assign) - self.func_ir._definitions[tuple_var.name] = [tuple_call] + self.state.func_ir._definitions[tuple_var.name] = [tuple_call] self._array_starts[lhs] = [self._set0_var] * ndims self._array_starts[lhs][0] = start_var self._array_counts[lhs] = new_size_list @@ -1408,7 +1387,7 @@ def _fix_1D_Var_alloc(self, size_var, lhs, scope, loc): new_size_var = None # size is single int var - if isinstance(size_var, ir.Var) and isinstance(self.typemap[size_var.name], types.Integer): + if isinstance(size_var, ir.Var) and isinstance(self.state.typemap[size_var.name], types.Integer): # array could be allocated inside 1D_Var nodes like sort if size_var.name not in self.oneDVar_len_vars: return [], size_var @@ -1417,9 +1396,9 @@ def _fix_1D_Var_alloc(self, size_var, lhs, scope, loc): def f(oneD_var_arr): # pragma: no cover arr_len = len(oneD_var_arr) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[arr_var.name],), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[arr_var.name],), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [arr_var]) out = f_block.body[:-3] # remove none return new_size_var = out[-1].target @@ -1429,7 +1408,7 @@ def f(oneD_var_arr): # pragma: no cover if isinstance(size_var, ir.Var): # see if size_var is a 1D_Var array's shape # it is already the local size, no need to transform - var_def = guard(get_definition, self.func_ir, size_var) + var_def = guard(get_definition, self.state.func_ir, size_var) oned_var_varnames = set(v for v in self._dist_analysis.array_dists if self._dist_analysis.array_dists[v] == Distribution.OneD_Var) if (isinstance(var_def, ir.Expr) and var_def.op == 'getattr' @@ -1439,7 +1418,7 @@ def f(oneD_var_arr): # pragma: no cover #assert size_var.name in self._tuple_table # self._tuple_table[size_var.name] size_list = self._get_tuple_varlist(size_var, out) - size_list = [ir_utils.convert_size_to_var(s, self.typemap, scope, loc, out) + size_list = [ir_utils.convert_size_to_var(s, self.state.typemap, scope, loc, out) for s in size_list] # tuple of int vars else: @@ -1450,9 +1429,9 @@ def f(oneD_var_arr): # pragma: no cover def f(oneD_var_arr): # pragma: no cover arr_len = len(oneD_var_arr) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[arr_var.name],), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[arr_var.name],), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [arr_var]) nodes = f_block.body[:-3] # remove none return new_size_var = nodes[-1].target @@ -1462,12 +1441,12 @@ def f(oneD_var_arr): # pragma: no cover new_size_list = copy.copy(size_list) new_size_list[0] = new_size_var tuple_var = ir.Var(scope, mk_unique_var("$tuple_var"), loc) - self.typemap[tuple_var.name] = types.containers.UniTuple( + self.state.typemap[tuple_var.name] = types.containers.UniTuple( types.intp, ndims) tuple_call = ir.Expr.build_tuple(new_size_list, loc) tuple_assign = ir.Assign(tuple_call, tuple_var, loc) out.append(tuple_assign) - self.func_ir._definitions[tuple_var.name] = [tuple_call] + self.state.func_ir._definitions[tuple_var.name] = [tuple_call] return out, tuple_var # new_body += self._run_1D_array_shape( @@ -1492,9 +1471,9 @@ def f(oneD_var_arr): # pragma: no cover # exec(func_text, {}, loc_vars) # f = loc_vars['f'] # - # f_ir = compile_to_numba_ir(f, {'np': np}, self.typingctx, - # (self.typemap[arr.name], types.intp), - # self.typemap, self.calltypes) + # f_ir = compile_to_numba_ir(f, {'np': np}, self.state.typingctx, + # (self.state.typemap[arr.name], types.intp), + # self.state.typemap, self.state.calltypes) # f_block = f_ir.blocks.popitem()[1] # replace_arg_nodes(f_block, [arr, dim1_size]) # nodes += f_block.body[:-3] @@ -1516,9 +1495,9 @@ def f(arr, dim1): # pragma: no cover sizes[0] = dim1 s = sizes.prod() - f_ir = compile_to_numba_ir(f, {'np': np}, self.typingctx, - (self.typemap[arr.name], types.intp), - self.typemap, self.calltypes) + f_ir = compile_to_numba_ir(f, {'np': np}, self.state.typingctx, + (self.state.typemap[arr.name], types.intp), + self.state.typemap, self.state.calltypes) f_block = f_ir.blocks.popitem()[1] replace_arg_nodes(f_block, [arr, dim1_size]) nodes += f_block.body[:-3] @@ -1535,13 +1514,13 @@ def _run_getsetitem(self, arr, index_var, node, full_node): #ndims = self._get_arr_ndim(arr.name) # if ndims==1: # multi-dimensional array could be indexed with 1D index - if isinstance(self.typemap[index_var.name], types.Integer): + if isinstance(self.state.typemap[index_var.name], types.Integer): sub_nodes = self._get_ind_sub( index_var, self._array_starts[arr.name][0]) out = sub_nodes _set_getsetitem_index(node, sub_nodes[-1].target) else: - index_list = guard(find_build_tuple, self.func_ir, index_var) + index_list = guard(find_build_tuple, self.state.func_ir, index_var) assert index_list is not None sub_nodes = self._get_ind_sub( index_list[0], self._array_starts[arr.name][0]) @@ -1549,7 +1528,7 @@ def _run_getsetitem(self, arr, index_var, node, full_node): new_index_list = copy.copy(index_list) new_index_list[0] = sub_nodes[-1].target tuple_var = ir.Var(scope, mk_unique_var("$tuple_var"), loc) - self.typemap[tuple_var.name] = self.typemap[index_var.name] + self.state.typemap[tuple_var.name] = self.state.typemap[index_var.name] tuple_call = ir.Expr.build_tuple(new_index_list, loc) tuple_assign = ir.Assign(tuple_call, tuple_var, loc) out.append(tuple_assign) @@ -1560,13 +1539,13 @@ def _run_getsetitem(self, arr, index_var, node, full_node): elif self._is_1D_arr(arr.name) and isinstance(node, (ir.StaticSetItem, ir.SetItem)): is_multi_dim = False # we only consider 1st dimension for multi-dim arrays - inds = guard(find_build_tuple, self.func_ir, index_var) + inds = guard(find_build_tuple, self.state.func_ir, index_var) if inds is not None: index_var = inds[0] is_multi_dim = True # no need for transformation for whole slices - if guard(is_whole_slice, self.typemap, self.func_ir, index_var): + if guard(is_whole_slice, self.state.typemap, self.state.func_ir, index_var): return out # TODO: support multi-dim slice setitem like X[a:b, c:d] @@ -1574,7 +1553,7 @@ def _run_getsetitem(self, arr, index_var, node, full_node): start = self._array_starts[arr.name][0] count = self._array_counts[arr.name][0] - if isinstance(self.typemap[index_var.name], types.Integer): + if isinstance(self.state.typemap[index_var.name], types.Integer): def f(A, val, index, chunk_start, chunk_count): # pragma: no cover hpat.distributed_lower._set_if_in_range( A, val, index, chunk_start, chunk_count) @@ -1582,7 +1561,7 @@ def f(A, val, index, chunk_start, chunk_count): # pragma: no cover return self._replace_func( f, [arr, node.value, index_var, start, count]) - assert isinstance(self.typemap[index_var.name], + assert isinstance(self.state.typemap[index_var.name], types.misc.SliceType), "slice index expected" # convert setitem with global range to setitem with local range @@ -1592,29 +1571,29 @@ def f(A, val, start, stop, chunk_start, chunk_count): # pragma: no cover start, stop, chunk_start, chunk_count) A[loc_start:loc_stop] = val - slice_call = get_definition(self.func_ir, index_var) + slice_call = get_definition(self.state.func_ir, index_var) slice_start = slice_call.args[0] slice_stop = slice_call.args[1] return self._replace_func( f, [arr, node.value, slice_start, slice_stop, start, count]) # print_node = ir.Print([start_var, end_var], None, loc) - # self.calltypes[print_node] = signature(types.none, types.int64, types.int64) + # self.state.calltypes[print_node] = signature(types.none, types.int64, types.int64) # out.append(print_node) # # setitem_attr_var = ir.Var(scope, mk_unique_var("$setitem_attr"), loc) # setitem_attr_call = ir.Expr.getattr(self._g_dist_var, "dist_setitem", loc) - # self.typemap[setitem_attr_var.name] = get_global_func_typ( + # self.state.typemap[setitem_attr_var.name] = get_global_func_typ( # distributed_api.dist_setitem) # setitem_assign = ir.Assign(setitem_attr_call, setitem_attr_var, loc) # out = [setitem_assign] # setitem_call = ir.Expr.call(setitem_attr_var, # [arr, index_var, node.value, start, count], (), loc) - # self.calltypes[setitem_call] = self.typemap[setitem_attr_var.name].get_call_type( - # self.typingctx, [self.typemap[arr.name], - # self.typemap[index_var.name], self.typemap[node.value.name], + # self.state.calltypes[setitem_call] = self.state.typemap[setitem_attr_var.name].get_call_type( + # self.state.typingctx, [self.state.typemap[arr.name], + # self.state.typemap[index_var.name], self.state.typemap[node.value.name], # types.intp, types.intp], {}) # err_var = ir.Var(scope, mk_unique_var("$setitem_err_var"), loc) - # self.typemap[err_var.name] = types.int32 + # self.state.typemap[err_var.name] = types.int32 # setitem_assign = ir.Assign(setitem_call, err_var, loc) # out.append(setitem_assign) @@ -1623,14 +1602,14 @@ def f(A, val, start, stop, chunk_start, chunk_count): # pragma: no cover lhs = full_node.target # we only consider 1st dimension for multi-dim arrays - inds = guard(find_build_tuple, self.func_ir, index_var) + inds = guard(find_build_tuple, self.state.func_ir, index_var) if inds is not None: index_var = inds[0] is_multi_dim = True - arr_def = guard(get_definition, self.func_ir, index_var) + arr_def = guard(get_definition, self.state.func_ir, index_var) if isinstance(arr_def, ir.Expr) and arr_def.op == 'call': - fdef = guard(find_callname, self.func_ir, arr_def, self.typemap) + fdef = guard(find_callname, self.state.func_ir, arr_def, self.state.typemap) if fdef == ('permutation', 'numpy.random'): self._array_starts[lhs.name] = self._array_starts[arr.name] self._array_counts[lhs.name] = self._array_counts[arr.name] @@ -1638,7 +1617,7 @@ def f(A, val, start, stop, chunk_start, chunk_count): # pragma: no cover out = self._run_permutation_array_index(lhs, arr, index_var) # no need for transformation for whole slices - if guard(is_whole_slice, self.typemap, self.func_ir, index_var): + if guard(is_whole_slice, self.state.typemap, self.state.func_ir, index_var): # A = X[:,3] self._array_starts[lhs.name] = [self._array_starts[arr.name][0]] self._array_counts[lhs.name] = [self._array_counts[arr.name][0]] @@ -1646,22 +1625,22 @@ def f(A, val, start, stop, chunk_start, chunk_count): # pragma: no cover # strided whole slice # e.g. A = X[::2,5] - elif guard(is_whole_slice, self.typemap, self.func_ir, index_var, accept_stride=True): + elif guard(is_whole_slice, self.state.typemap, self.state.func_ir, index_var, accept_stride=True): # FIXME: we use rebalance array to handle the output array # TODO: convert to neighbor exchange # on each processor, the slice has to start from an offset: # |step-(start%step)| in_arr = full_node.value.value start = self._array_starts[in_arr.name][0] - step = get_slice_step(self.typemap, self.func_ir, index_var) + step = get_slice_step(self.state.typemap, self.state.func_ir, index_var) def f(A, start, step): offset = abs(step - (start % step)) % step B = A[offset::step] - f_block = compile_to_numba_ir(f, {}, self.typingctx, - (self.typemap[in_arr.name], types.intp, types.intp), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {}, self.state.typingctx, + (self.state.typemap[in_arr.name], types.intp, types.intp), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [in_arr, start, step]) out = f_block.body[:-3] # remove none return imb_arr = out[-1].target @@ -1672,7 +1651,7 @@ def f(A, start, step): out[-1].target = lhs elif self._is_REP(lhs.name) and guard( - is_const_slice, self.typemap, self.func_ir, index_var): + is_const_slice, self.state.typemap, self.state.func_ir, index_var): # cases like S.head() # bcast if all in rank 0, otherwise gatherv in_arr = full_node.value.value @@ -1686,7 +1665,7 @@ def f(A, start, step): def _run_parfor(self, parfor, namevar_table): # stencil_accesses, neighborhood = get_stencil_accesses( - # parfor, self.typemap) + # parfor, self.state.typemap) # Thread and 1D parfors turn to gufunc in multithread mode if (hpat.multithread_mode @@ -1715,13 +1694,13 @@ def _run_parfor(self, parfor, namevar_table): # if right_length: # new_range_size = ir.Var( # scope, mk_unique_var("new_range_size"), loc) - # self.typemap[new_range_size.name] = types.intp + # self.state.typemap[new_range_size.name] = types.intp # index_const = ir.Var(scope, mk_unique_var("index_const"), loc) - # self.typemap[index_const.name] = types.intp + # self.state.typemap[index_const.name] = types.intp # out.append( # ir.Assign(ir.Const(right_length, loc), index_const, loc)) # calc_call = ir.Expr.binop('+', range_size, index_const, loc) - # self.calltypes[calc_call] = ir_utils.find_op_typ('+', + # self.state.calltypes[calc_call] = ir_utils.find_op_typ('+', # [types.intp, types.intp]) # out.append(ir.Assign(calc_call, new_range_size, loc)) # range_size = new_range_size @@ -1730,7 +1709,7 @@ def _run_parfor(self, parfor, namevar_table): "$loop", "get_end", distributed_api.get_end) out += div_nodes # print_node = ir.Print([start_var, end_var, range_size], None, loc) - # self.calltypes[print_node] = signature(types.none, types.int64, types.int64, types.intp) + # self.state.calltypes[print_node] = signature(types.none, types.int64, types.int64, types.intp) # out.append(print_node) parfor.loop_nests[0].start = start_var @@ -1762,9 +1741,9 @@ def _run_parfor_1D_Var(self, parfor, namevar_table): def f(A): # pragma: no cover arr_len = len(A) - f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[arr_var.name],), - self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[arr_var.name],), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [arr_var]) nodes = f_block.body[:-3] # remove none return l.stop = nodes[-1].target @@ -1789,7 +1768,7 @@ def f(A): # pragma: no cover loc = l_nest.index_variable.loc if isinstance(l_nest.start, int): start_var = ir.Var(scope, mk_unique_var("loop_start"), loc) - self.typemap[start_var.name] = types.intp + self.state.typemap[start_var.name] = types.intp prepend.append(ir.Assign( ir.Const(l_nest.start, loc), start_var, loc)) l_nest.start = start_var @@ -1801,13 +1780,13 @@ def _fix_ind_bounds(start, stop): return start + prefix, stop + prefix f_block = compile_to_numba_ir(_fix_ind_bounds, {'hpat': hpat}, - self.typingctx, (types.intp, types.intp), self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, (types.intp, types.intp), self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [l_nest.start, l_nest.stop]) nodes = f_block.body[:-2] ret_var = nodes[-1].target - gen_getitem(l_nest.start, ret_var, 0, self.calltypes, nodes) - gen_getitem(l_nest.stop, ret_var, 1, self.calltypes, nodes) + gen_getitem(l_nest.start, ret_var, 0, self.state.calltypes, nodes) + gen_getitem(l_nest.stop, ret_var, 1, self.state.calltypes, nodes) prepend += nodes array_accesses = ir_utils.get_array_accesses(parfor.loop_body) @@ -1825,12 +1804,12 @@ def _run_arg(self, assign): rhs = assign.value out = [assign] - if rhs.name not in self.metadata['distributed']: + if rhs.name not in self.state.metadata['distributed']: return None arr = assign.target - typ = self.typemap[arr.name] - if is_array_container(self.typemap, arr.name): + typ = self.state.typemap[arr.name] + if is_array_container(self.state.typemap, arr.name): return None # TODO: comprehensive support for Series vars @@ -1840,7 +1819,7 @@ def _run_arg(self, assign): # gen len() using 1D_Var reduce approach. # TODO: refactor to avoid reduction - ndim = self.typemap[arr.name].ndim + ndim = self.state.typemap[arr.name].ndim out += self._gen_1D_Var_len(arr) total_length = out[-1].target div_nodes, start_var, count_var = self._gen_1D_div( @@ -1861,7 +1840,7 @@ def _index_has_par_index(self, index, other_index): if index == other_index: return True # multi-dim case - tup_list = guard(find_build_tuple, self.func_ir, index) + tup_list = guard(find_build_tuple, self.state.func_ir, index) if tup_list is not None: index_tuple = [var.name for var in tup_list] if index_tuple[0] == index: @@ -1874,7 +1853,7 @@ def _gen_parfor_reductions(self, parfor, namevar_table): pre = [] out = [] _, reductions = get_parfor_reductions( - parfor, parfor.params, self.calltypes) + parfor, parfor.params, self.state.calltypes) for reduce_varname, (init_val, reduce_nodes) in reductions.items(): reduce_op = guard(self._get_reduce_op, reduce_nodes) @@ -1888,7 +1867,7 @@ def _gen_parfor_reductions(self, parfor, namevar_table): # def _get_var_const_val(self, var): # if isinstance(var, int): # return var - # node = guard(get_definition, self.func_ir, var) + # node = guard(get_definition, self.state.func_ir, var) # if isinstance(node, ir.Const): # return node.value # if isinstance(node, ir.Expr): @@ -1910,7 +1889,7 @@ def _gen_1D_div(self, size_var, scope, loc, prefix, end_call_name, end_call): if isinstance(size_var, int): new_size_var = ir.Var( scope, mk_unique_var(prefix + "_size_var"), loc) - self.typemap[new_size_var.name] = types.int64 + self.state.typemap[new_size_var.name] = types.int64 size_assign = ir.Assign(ir.Const(size_var, loc), new_size_var, loc) div_nodes.append(size_assign) size_var = new_size_var @@ -1918,42 +1897,42 @@ def _gen_1D_div(self, size_var, scope, loc, prefix, end_call_name, end_call): # attr call: start_attr = getattr(g_dist_var, get_start) start_attr_call = ir.Expr.getattr(self._g_dist_var, "get_start", loc) start_attr_var = ir.Var(scope, mk_unique_var("$get_start_attr"), loc) - self.typemap[start_attr_var.name] = get_global_func_typ( + self.state.typemap[start_attr_var.name] = get_global_func_typ( distributed_api.get_start) start_attr_assign = ir.Assign(start_attr_call, start_attr_var, loc) # start_var = get_start(size, rank, pes) start_var = ir.Var(scope, mk_unique_var(prefix + "_start_var"), loc) - self.typemap[start_var.name] = types.int64 + self.state.typemap[start_var.name] = types.int64 start_expr = ir.Expr.call(start_attr_var, [size_var, self._size_var, self._rank_var], (), loc) - self.calltypes[start_expr] = self.typemap[start_attr_var.name].get_call_type( - self.typingctx, [types.int64, types.int32, types.int32], {}) + self.state.calltypes[start_expr] = self.state.typemap[start_attr_var.name].get_call_type( + self.state.typingctx, [types.int64, types.int32, types.int32], {}) start_assign = ir.Assign(start_expr, start_var, loc) # attr call: end_attr = getattr(g_dist_var, get_end) end_attr_call = ir.Expr.getattr(self._g_dist_var, end_call_name, loc) end_attr_var = ir.Var(scope, mk_unique_var("$get_end_attr"), loc) - self.typemap[end_attr_var.name] = get_global_func_typ(end_call) + self.state.typemap[end_attr_var.name] = get_global_func_typ(end_call) end_attr_assign = ir.Assign(end_attr_call, end_attr_var, loc) end_var = ir.Var(scope, mk_unique_var(prefix + "_end_var"), loc) - self.typemap[end_var.name] = types.int64 + self.state.typemap[end_var.name] = types.int64 end_expr = ir.Expr.call(end_attr_var, [size_var, self._size_var, self._rank_var], (), loc) - self.calltypes[end_expr] = self.typemap[end_attr_var.name].get_call_type( - self.typingctx, [types.int64, types.int32, types.int32], {}) + self.state.calltypes[end_expr] = self.state.typemap[end_attr_var.name].get_call_type( + self.state.typingctx, [types.int64, types.int32, types.int32], {}) end_assign = ir.Assign(end_expr, end_var, loc) div_nodes += [start_attr_assign, start_assign, end_attr_assign, end_assign] return div_nodes, start_var, end_var def _get_ind_sub(self, ind_var, start_var): if (isinstance(ind_var, slice) - or isinstance(self.typemap[ind_var.name], + or isinstance(self.state.typemap[ind_var.name], types.misc.SliceType)): return self._get_ind_sub_slice(ind_var, start_var) # gen sub - f_ir = compile_to_numba_ir(lambda ind, start: ind - start, {}, self.typingctx, - (types.intp, types.intp), self.typemap, self.calltypes) + f_ir = compile_to_numba_ir(lambda ind, start: ind - start, {}, self.state.typingctx, + (types.intp, types.intp), self.state.typemap, self.state.calltypes) block = f_ir.blocks.popitem()[1] replace_arg_nodes(block, [ind_var, start_var]) return block.body[:-2] @@ -1972,10 +1951,10 @@ def _get_ind_sub_slice(self, slice_var, offset_var): def f(old_slice, offset): # pragma: no cover return slice(old_slice.start - offset, old_slice.stop - offset) args = [slice_var, offset_var] - slice_type = self.typemap[slice_var.name] + slice_type = self.state.typemap[slice_var.name] arg_typs = (slice_type, types.intp,) - _globals = self.func_ir.func_id.func.__globals__ - f_ir = compile_to_numba_ir(f, _globals, self.typingctx, arg_typs, self.typemap, self.calltypes) + _globals = self.state.func_ir.func_id.func.__globals__ + f_ir = compile_to_numba_ir(f, _globals, self.state.typingctx, arg_typs, self.state.typemap, self.state.calltypes) _, block = f_ir.blocks.popitem() replace_arg_nodes(block, args) return block.body[:-2] # ignore return nodes @@ -1996,11 +1975,11 @@ def _dist_prints(self, blocks): prev_block.body = block.body[:i] rank_comp_var = ir.Var(scope, mk_unique_var("$rank_comp"), loc) - self.typemap[rank_comp_var.name] = types.boolean + self.state.typemap[rank_comp_var.name] = types.boolean comp_expr = ir.Expr.binop(operator.eq, self._rank_var, self._set0_var, loc) - expr_typ = self.typingctx.resolve_function_type(operator.eq, (types.int32, types.int64), {}) + expr_typ = self.state.typingctx.resolve_function_type(operator.eq, (types.int32, types.int64), {}) #expr_typ = find_op_typ(operator.eq, [types.int32, types.int64]) - self.calltypes[comp_expr] = expr_typ + self.state.calltypes[comp_expr] = expr_typ comp_assign = ir.Assign(comp_expr, rank_comp_var, loc) prev_block.body.append(comp_assign) print_branch = ir.Branch(rank_comp_var, print_label, block_label, loc) @@ -2019,13 +1998,13 @@ def _dist_prints(self, blocks): def _file_open_set_parallel(self, file_varname): var = file_varname while True: - var_def = get_definition(self.func_ir, var) + var_def = get_definition(self.state.func_ir, var) require(isinstance(var_def, ir.Expr)) if var_def.op == 'call': - fdef = find_callname(self.func_ir, var_def) + fdef = find_callname(self.state.func_ir, var_def) if (fdef[0] in ('create_dataset', 'create_group') and isinstance(fdef[1], ir.Var) - and self.typemap[fdef[1].name] in (h5file_type, h5group_type)): + and self.state.typemap[fdef[1].name] in (h5file_type, h5group_type)): self._file_open_set_parallel(fdef[1].name) return else: @@ -2036,7 +2015,7 @@ def _file_open_set_parallel(self, file_varname): require(var_def.op in ('getitem', 'static_getitem')) var = var_def.value.name - # for label, block in self.func_ir.blocks.items(): + # for label, block in self.state.func_ir.blocks.items(): # for stmt in block.body: # if (isinstance(stmt, ir.Assign) # and stmt.target.name == file_varname): @@ -2056,20 +2035,20 @@ def _gen_barrier(self): def f(): # pragma: no cover return hpat.distributed_api.barrier() - f_blocks = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, {}, self.typemap, self.calltypes).blocks + f_blocks = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, {}, self.state.typemap, self.state.calltypes).blocks block = f_blocks[min(f_blocks.keys())] return block.body[:-2] # remove return def _gen_reduce(self, reduce_var, reduce_op, scope, loc): op_var = ir.Var(scope, mk_unique_var("$reduce_op"), loc) - self.typemap[op_var.name] = types.int32 + self.state.typemap[op_var.name] = types.int32 op_assign = ir.Assign(ir.Const(reduce_op.value, loc), op_var, loc) def f(val, op): # pragma: no cover hpat.distributed_api.dist_reduce(val, op) - f_ir = compile_to_numba_ir(f, {'hpat': hpat}, self.typingctx, - (self.typemap[reduce_var.name], types.int32), self.typemap, self.calltypes) + f_ir = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, + (self.state.typemap[reduce_var.name], types.int32), self.state.typemap, self.state.calltypes) _, block = f_ir.blocks.popitem() replace_arg_nodes(block, [reduce_var, op_var]) @@ -2094,13 +2073,13 @@ def _get_reduce_op(self, reduce_nodes): return Reduce_Type.Prod if rhs.op == 'call': - func = find_callname(self.func_ir, rhs, self.typemap) + func = find_callname(self.state.func_ir, rhs, self.state.typemap) if func == ('min', 'builtins'): - if isinstance(self.typemap[rhs.args[0].name], numba.typing.builtins.IndexValueType): + if isinstance(self.state.typemap[rhs.args[0].name], numba.typing.builtins.IndexValueType): return Reduce_Type.Argmin return Reduce_Type.Min if func == ('max', 'builtins'): - if isinstance(self.typemap[rhs.args[0].name], numba.typing.builtins.IndexValueType): + if isinstance(self.state.typemap[rhs.args[0].name], numba.typing.builtins.IndexValueType): return Reduce_Type.Argmax return Reduce_Type.Max @@ -2110,9 +2089,9 @@ def _gen_init_reduce(self, reduce_var, reduce_op): """generate code to initialize reduction variables on non-root processors. """ - red_var_typ = self.typemap[reduce_var.name] + red_var_typ = self.state.typemap[reduce_var.name] el_typ = red_var_typ - if is_np_array(self.typemap, reduce_var.name): + if is_np_array(self.state.typemap, reduce_var.name): el_typ = red_var_typ.dtype init_val = None pre_init_val = "" @@ -2132,17 +2111,17 @@ def _gen_init_reduce(self, reduce_var, reduce_op): assert init_val is not None - if is_np_array(self.typemap, reduce_var.name): + if is_np_array(self.state.typemap, reduce_var.name): pre_init_val = "v = np.full_like(s, {}, s.dtype)".format(init_val) init_val = "v" f_text = "def f(s):\n {}\n s = hpat.distributed_lower._root_rank_select(s, {})".format(pre_init_val, init_val) loc_vars = {} - exec(f_text, {}, loc_vars) + exec(f_text, {'hpat': hpat}, loc_vars) f = loc_vars['f'] f_block = compile_to_numba_ir(f, {'hpat': hpat, 'numba': numba, 'np': np}, - self.typingctx, (red_var_typ,), self.typemap, self.calltypes).blocks.popitem()[1] + self.state.typingctx, (red_var_typ,), self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [reduce_var]) nodes = f_block.body[:-3] nodes[-1].target = reduce_var @@ -2152,11 +2131,11 @@ def _get_tuple_varlist(self, tup_var, out): """ get the list of variables that hold values in the tuple variable. add node to out if code generation needed. """ - t_list = guard(find_build_tuple, self.func_ir, tup_var) + t_list = guard(find_build_tuple, self.state.func_ir, tup_var) if t_list is not None: return t_list - assert isinstance(self.typemap[tup_var.name], types.UniTuple) - ndims = self.typemap[tup_var.name].count + assert isinstance(self.state.typemap[tup_var.name], types.UniTuple) + ndims = self.state.typemap[tup_var.name].count f_text = "def f(tup_var):\n" for i in range(ndims): f_text += " val{} = tup_var[{}]\n".format(i, i) @@ -2165,11 +2144,11 @@ def _get_tuple_varlist(self, tup_var, out): f = loc_vars['f'] f_block = compile_to_numba_ir(f, {}, - self.typingctx, - (self.typemap[tup_var.name], + self.state.typingctx, + (self.state.typemap[tup_var.name], ), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [tup_var]) nodes = f_block.body[:-3] vals_list = [] @@ -2202,11 +2181,11 @@ def _replace_func(self, func, args, const=False, pre_nodes=None, extra_globals=N glbls = {'numba': numba, 'np': np, 'hpat': hpat} if extra_globals is not None: glbls.update(extra_globals) - arg_typs = tuple(self.typemap[v.name] for v in args) + arg_typs = tuple(self.state.typemap[v.name] for v in args) if const: new_args = [] for i, arg in enumerate(args): - val = guard(find_const, self.func_ir, arg) + val = guard(find_const, self.state.func_ir, arg) if val: new_args.append(types.literal(val)) else: @@ -2215,9 +2194,9 @@ def _replace_func(self, func, args, const=False, pre_nodes=None, extra_globals=N return ReplaceFunc(func, arg_typs, args, glbls, pre_nodes) def _get_arr_ndim(self, arrname): - if self.typemap[arrname] == string_array_type: + if self.state.typemap[arrname] == string_array_type: return 1 - return self.typemap[arrname].ndim + return self.state.typemap[arrname].ndim def _is_1D_arr(self, arr_name): # some arrays like stencil buffers are added after analysis so diff --git a/hpat/hiframes/aggregate.py b/hpat/hiframes/aggregate.py index 7ac900e48..541321768 100644 --- a/hpat/hiframes/aggregate.py +++ b/hpat/hiframes/aggregate.py @@ -6,7 +6,7 @@ import copy import numpy as np import numba -from numba import typeinfer, ir, ir_utils, config, types, compiler +from numba import typeinfer, ir, ir_utils, config, types, compiler, typed_passes from numba.ir_utils import (visit_vars_inner, replace_vars_inner, remove_dead, compile_to_numba_ir, replace_arg_nodes, replace_vars_stmt, find_callname, guard, @@ -1047,51 +1047,54 @@ def gen_top_level_agg_func(key_names, return_key, red_var_typs, out_typs, def compile_to_optimized_ir(func, arg_typs, typingctx): + state = namedtuple('State', + ['typingctx', 'targetctx', 'args', 'func_ir', 'typemap', 'return_type', + 'calltypes', 'metadata']) + # XXX are outside function's globals needed? code = func.code if hasattr(func, 'code') else func.__code__ - f_ir = get_ir_of_code({'numba': numba, 'np': np, 'hpat': hpat}, code) + state.func_ir = get_ir_of_code({'numba': numba, 'np': np, 'hpat': hpat}, code) + + state.typingctx = typingctx + state.args = arg_typs + state.locals = {} + state.metadata = {} # rename all variables to avoid conflict (init and eval nodes) - var_table = get_name_var_table(f_ir.blocks) + var_table = get_name_var_table(state.func_ir.blocks) new_var_dict = {} for name, _ in var_table.items(): new_var_dict[name] = mk_unique_var(name) - replace_var_names(f_ir.blocks, new_var_dict) - f_ir._definitions = build_definitions(f_ir.blocks) + replace_var_names(state.func_ir.blocks, new_var_dict) + state.func_ir._definitions = build_definitions(state.func_ir.blocks) - assert f_ir.arg_count == 1, "agg function should have one input" - input_name = f_ir.arg_names[0] - df_pass = hpat.hiframes.hiframes_untyped.HiFrames( - f_ir, typingctx, arg_typs, {}, {}) - df_pass.run() - remove_dead(f_ir.blocks, f_ir.arg_names, f_ir) - typemap, return_type, calltypes = compiler.type_inference_stage( - typingctx, f_ir, arg_typs, None) + assert state.func_ir.arg_count == 1, "agg function should have one input" + input_name = state.func_ir.arg_names[0] + df_pass = hpat.hiframes.hiframes_untyped.HiFramesPass() + df_pass.run_pass(state) + remove_dead(state.func_ir.blocks, state.func_ir.arg_names, state.func_ir) + state.typemap, return_type, state.calltypes = typed_passes.type_inference_stage( + typingctx, state.func_ir, arg_typs, None) options = numba.targets.cpu.ParallelOptions(True) flags = compiler.Flags() - targetctx = numba.targets.cpu.CPUContext(typingctx) - - DummyPipeline = namedtuple('DummyPipeline', - ['typingctx', 'targetctx', 'args', 'func_ir', 'typemap', 'return_type', - 'calltypes']) - pm = DummyPipeline(typingctx, targetctx, None, f_ir, typemap, return_type, - calltypes) - preparfor_pass = numba.parfor.PreParforPass(f_ir, typemap, calltypes, typingctx, options) + state.targetctx = numba.targets.cpu.CPUContext(typingctx) + + preparfor_pass = numba.parfor.PreParforPass(state.func_ir, state.typemap, state.calltypes, state.typingctx, options) preparfor_pass.run() - f_ir._definitions = build_definitions(f_ir.blocks) - df_t_pass = hpat.hiframes.hiframes_typed.HiFramesTyped(f_ir, typingctx, typemap, calltypes) - df_t_pass.run() - numba.rewrites.rewrite_registry.apply('after-inference', pm, f_ir) - parfor_pass = numba.parfor.ParforPass(f_ir, typemap, - calltypes, return_type, typingctx, + state.func_ir._definitions = build_definitions(state.func_ir.blocks) + df_t_pass = hpat.hiframes.hiframes_typed.HiFramesTypedPass() + df_t_pass.run_pass(state) + numba.rewrites.rewrite_registry.apply('after-inference', state) + parfor_pass = numba.parfor.ParforPass(state.func_ir, state.typemap, + state.calltypes, return_type, state.typingctx, options, flags) parfor_pass.run() - remove_dels(f_ir.blocks) + remove_dels(state.func_ir.blocks) # make sure eval nodes are after the parfor for easier extraction # TODO: extract an eval func more robustly - numba.parfor.maximize_fusion(f_ir, f_ir.blocks, typemap, False) - return f_ir, pm + numba.parfor.maximize_fusion(state.func_ir, state.func_ir.blocks, state.typemap, False) + return state.func_ir, state def get_agg_func_struct(agg_func, in_col_types, out_col_typs, typingctx, diff --git a/hpat/hiframes/dataframe_pass.py b/hpat/hiframes/dataframe_pass.py index c7c774b2f..20ee75daa 100644 --- a/hpat/hiframes/dataframe_pass.py +++ b/hpat/hiframes/dataframe_pass.py @@ -17,10 +17,11 @@ from numba.typing.arraydecl import ArrayAttribute from numba.extending import overload from numba.typing.templates import infer_global, AbstractTemplate, signature +from numba.compiler_machinery import FunctionPass, register_pass import hpat from hpat import hiframes from hpat.utils import (debug_prints, inline_new_blocks, ReplaceFunc, - is_whole_slice, is_array, is_assign, sanitize_varname) + is_whole_slice, is_array, is_assign, sanitize_varname, update_globals) from hpat.str_ext import string_type from hpat.str_arr_ext import (string_array_type, StringArrayType, is_str_arr_typ, pre_alloc_string_array) @@ -36,23 +37,33 @@ from hpat.hiframes.aggregate import get_agg_func -class DataFramePass(object): +@register_pass(mutates_CFG=True, analysis_only=False) +class DataFramePass(FunctionPass): """Analyze and transform dataframe calls after typing""" - def __init__(self, func_ir, typingctx, typemap, calltypes): - self.func_ir = func_ir - self.typingctx = typingctx - self.typemap = typemap - self.calltypes = calltypes + _name = "dataframe_pass" - def run(self): - blocks = self.func_ir.blocks + def __init__(self): + pass + + def run_pass(self, state): + return DataFramePassImpl(state).run_pass() + + +class DataFramePassImpl(object): + + def __init__(self, state): + self.state = state + + def run_pass(self): + blocks = self.state.func_ir.blocks # topo_order necessary so DataFrame data replacement optimization can # be performed in one pass topo_order = find_topo_order(blocks) work_list = list((l, blocks[l]) for l in reversed(topo_order)) dead_labels = [] while work_list: + label, block = work_list.pop() if label in dead_labels: continue @@ -63,31 +74,31 @@ def run(self): branch_or_jump = block.body[-1] if isinstance(branch_or_jump, ir.Branch): branch = branch_or_jump - cond_val = guard(_eval_const_var, self.func_ir, branch.cond) + cond_val = guard(_eval_const_var, self.state.func_ir, branch.cond) if cond_val is not None: # replace branch with Jump dead_label = branch.falsebr if cond_val else branch.truebr jmp_label = branch.truebr if cond_val else branch.falsebr jmp = ir.Jump(jmp_label, branch.loc) block.body[-1] = jmp - cfg = compute_cfg_from_blocks(self.func_ir.blocks) + cfg = compute_cfg_from_blocks(self.state.func_ir.blocks) if dead_label in cfg.dead_nodes(): dead_labels.append(dead_label) # remove definitions in dead block so const variables can # be found later (pd.merge() example) # TODO: add this to dead_branch_prune pass - for inst in self.func_ir.blocks[dead_label].body: + for inst in self.state.func_ir.blocks[dead_label].body: if is_assign(inst): - self.func_ir._definitions[inst.target.name].remove( + self.state.func_ir._definitions[inst.target.name].remove( inst.value) - del self.func_ir.blocks[dead_label] + del self.state.func_ir.blocks[dead_label] else: # the jmp block overrides some definitions of current # block so remove dead defs and update _definitions # example: test_join_left_seq1 jmp_defs = set() - for inst in self.func_ir.blocks[jmp_label].body: + for inst in self.state.func_ir.blocks[jmp_label].body: if is_assign(inst): jmp_defs.add(inst.target.name) used_vars = set() @@ -96,7 +107,7 @@ def run(self): if (is_assign(inst) and inst.target.name not in used_vars and inst.target.name in jmp_defs): - self.func_ir._definitions[inst.target.name].remove(inst.value) + self.state.func_ir._definitions[inst.target.name].remove(inst.value) continue used_vars.update(v.name for v in inst.list_vars()) new_body.append(inst) @@ -109,7 +120,7 @@ def run(self): out_nodes = [inst] if isinstance(inst, ir.Assign): - self.func_ir._definitions[inst.target.name].remove(inst.value) + self.state.func_ir._definitions[inst.target.name].remove(inst.value) out_nodes = self._run_assign(inst) elif isinstance(inst, (ir.SetItem, ir.StaticSetItem)): out_nodes = self._run_setitem(inst) @@ -129,17 +140,18 @@ def run(self): ir.Var(block.scope, "dummy", inst.loc), rp_func.args, (), inst.loc) block.body = new_body + block.body[i:] - callee_blocks = inline_closure_call(self.func_ir, rp_func.glbls, - block, len(new_body), rp_func.func, self.typingctx, + update_globals(rp_func.func, rp_func.glbls) + callee_blocks = inline_closure_call(self.state.func_ir, rp_func.glbls, + block, len(new_body), rp_func.func, self.state.typingctx, rp_func.arg_types, - self.typemap, self.calltypes) + self.state.typemap, self.state.calltypes) # TODO: remove after Numba #3946 is merged if isinstance(callee_blocks, tuple): callee_blocks = callee_blocks[0] # add blocks in reversed topo order to enable dead branch # pruning (merge example) # TODO: fix inline_closure_call() - topo_order = find_topo_order(self.func_ir.blocks) + topo_order = find_topo_order(self.state.func_ir.blocks) for c_label in reversed(topo_order): if c_label in callee_blocks: c_block = callee_blocks[c_label] @@ -152,27 +164,28 @@ def run(self): if (target_label not in callee_blocks and target_label not in work_list): work_list.append((target_label, - self.func_ir.blocks[target_label])) + self.state.func_ir.blocks[target_label])) work_list.append((c_label, c_block)) replaced = True break if isinstance(out_nodes, dict): block.body = new_body + block.body[i:] - inline_new_blocks(self.func_ir, block, i, out_nodes, work_list) + inline_new_blocks(self.state.func_ir, block, i, out_nodes, work_list) replaced = True break if not replaced: blocks[label].body = new_body - self.func_ir.blocks = ir_utils.simplify_CFG(self.func_ir.blocks) - while ir_utils.remove_dead(self.func_ir.blocks, self.func_ir.arg_names, - self.func_ir, self.typemap): + self.state.func_ir.blocks = ir_utils.simplify_CFG(self.state.func_ir.blocks) + while ir_utils.remove_dead(self.state.func_ir.blocks, self.state.func_ir.arg_names, + self.state.func_ir, self.state.typemap): pass - self.func_ir._definitions = build_definitions(self.func_ir.blocks) - dprint_func_ir(self.func_ir, "after hiframes_typed") - return + self.state.func_ir._definitions = build_definitions(self.state.func_ir.blocks) + dprint_func_ir(self.state.func_ir, "after hiframes_typed") + + return True def _run_assign(self, assign): lhs = assign.target @@ -213,14 +226,14 @@ def _run_getitem(self, assign, rhs): assert rhs.op == 'static_getitem' index_typ = numba.typeof(rhs.index) index_var = ir.Var(lhs.scope, mk_unique_var('dummy_index'), lhs.loc) - self.typemap[index_var.name] = index_typ + self.state.typemap[index_var.name] = index_typ else: - index_typ = self.typemap[index_var.name] + index_typ = self.state.typemap[index_var.name] # A = df['column'] or df[['C1', 'C2']] if rhs.op == 'static_getitem' and self._is_df_var(rhs.value): df_var = rhs.value - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] index = rhs.index # A = df['column'] @@ -261,7 +274,7 @@ def _run_getitem(self, assign, rhs): or self.is_int_list_or_arr(index_var.name) or isinstance(index_typ, types.SliceType))): # TODO: check for errors - df_var = guard(get_definition, self.func_ir, rhs.value).value + df_var = guard(get_definition, self.state.func_ir, rhs.value).value return self._gen_df_filter(df_var, index_var, lhs) # df.iloc[1:n,0], df.loc[1:n,'A'] @@ -269,9 +282,9 @@ def _run_getitem(self, assign, rhs): and isinstance(index_typ, types.Tuple) and len(index_typ) == 2): # - df_var = guard(get_definition, self.func_ir, rhs.value).value - df_typ = self.typemap[df_var.name] - ind_def = guard(get_definition, self.func_ir, index_var) + df_var = guard(get_definition, self.state.func_ir, rhs.value).value + df_typ = self.state.typemap[df_var.name] + ind_def = guard(get_definition, self.state.func_ir, index_var) # TODO check and report errors assert isinstance(ind_def, ir.Expr) and ind_def.op == 'build_tuple' @@ -281,19 +294,19 @@ def _run_getitem(self, assign, rhs): if rhs.op == 'static_getitem': col_ind = rhs.index[1] else: - col_ind = guard(find_const, self.func_ir, ind_def.items[1]) + col_ind = guard(find_const, self.state.func_ir, ind_def.items[1]) col_name = df_typ.columns[col_ind] else: # df.loc - col_name = guard(find_const, self.func_ir, ind_def.items[1]) + col_name = guard(find_const, self.state.func_ir, ind_def.items[1]) col_filter_var = ind_def.items[0] name_var = ir.Var(lhs.scope, mk_unique_var('df_col_name'), lhs.loc) - self.typemap[name_var.name] = types.StringLiteral(col_name) + self.state.typemap[name_var.name] = types.StringLiteral(col_name) nodes.append( ir.Assign(ir.Const(col_name, lhs.loc), name_var, lhs.loc)) in_arr = self._get_dataframe_data(df_var, col_name, nodes) - if guard(is_whole_slice, self.typemap, self.func_ir, col_filter_var): + if guard(is_whole_slice, self.state.typemap, self.state.func_ir, col_filter_var): def func(A, ind, name): return hpat.hiframes.api.init_series(A, None, name) else: @@ -304,8 +317,8 @@ def func(A, ind, name): return self._replace_func(func, [in_arr, col_filter_var, name_var], pre_nodes=nodes) if self._is_df_iat_var(rhs.value): - df_var = guard(get_definition, self.func_ir, rhs.value).value - df_typ = self.typemap[df_var.name] + df_var = guard(get_definition, self.state.func_ir, rhs.value).value + df_typ = self.state.typemap[df_var.name] # df.iat[3,1] if (rhs.op == 'static_getitem' and isinstance(rhs.index, tuple) and len(rhs.index) == 2 and isinstance(rhs.index[0], int) @@ -319,8 +332,8 @@ def func(A, ind, name): # df.iat[n,1] if isinstance(index_typ, types.Tuple) and len(index_typ) == 2: - ind_def = guard(get_definition, self.func_ir, index_var) - col_ind = guard(find_const, self.func_ir, ind_def.items[1]) + ind_def = guard(get_definition, self.state.func_ir, index_var) + col_ind = guard(find_const, self.state.func_ir, ind_def.items[1]) col_name = df_typ.columns[col_ind] in_arr = self._get_dataframe_data(df_var, col_name, nodes) row_ind = ind_def.items[0] @@ -332,13 +345,13 @@ def func(A, ind, name): def _gen_df_filter(self, df_var, index_var, lhs): nodes = [] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] in_vars = {} out_vars = {} for col in df_typ.columns: in_arr = self._get_dataframe_data(df_var, col, nodes) out_arr = ir.Var(lhs.scope, mk_unique_var(col), lhs.loc) - self.typemap[out_arr.name] = self.typemap[in_arr.name] + self.state.typemap[out_arr.name] = self.state.typemap[in_arr.name] in_vars[col] = in_arr out_vars[col] = out_arr @@ -351,20 +364,20 @@ def _gen_df_filter(self, df_var, index_var, lhs): _init_df, list(out_vars.values()), pre_nodes=nodes) def _run_setitem(self, inst): - target_typ = self.typemap[inst.target.name] + target_typ = self.state.typemap[inst.target.name] nodes = [] index_var = (inst.index_var if isinstance(inst, ir.StaticSetItem) else inst.index) - index_typ = self.typemap[index_var.name] + index_typ = self.state.typemap[index_var.name] if self._is_df_iat_var(inst.target): - df_var = guard(get_definition, self.func_ir, inst.target).value - df_typ = self.typemap[df_var.name] + df_var = guard(get_definition, self.state.func_ir, inst.target).value + df_typ = self.state.typemap[df_var.name] val = inst.value # df.iat[n,1] = 3 if isinstance(index_typ, types.Tuple) and len(index_typ) == 2: - ind_def = guard(get_definition, self.func_ir, index_var) - col_ind = guard(find_const, self.func_ir, ind_def.items[1]) + ind_def = guard(get_definition, self.state.func_ir, index_var) + col_ind = guard(find_const, self.state.func_ir, ind_def.items[1]) col_name = df_typ.columns[col_ind] in_arr = self._get_dataframe_data(df_var, col_name, nodes) row_ind = ind_def.items[0] @@ -376,7 +389,7 @@ def _impl(A, row_ind, val): return [inst] def _run_getattr(self, assign, rhs): - rhs_type = self.typemap[rhs.value.name] # get type of rhs value "df" + rhs_type = self.state.typemap[rhs.value.name] # get type of rhs value "df" # S = df.A (get dataframe column) # TODO: check invalid df.Attr? @@ -386,7 +399,7 @@ def _run_getattr(self, assign, rhs): arr = self._get_dataframe_data(rhs.value, col_name, nodes) index = self._get_dataframe_index(rhs.value, nodes) name = ir.Var(arr.scope, mk_unique_var('df_col_name'), arr.loc) - self.typemap[name.name] = types.StringLiteral(col_name) + self.state.typemap[name.name] = types.StringLiteral(col_name) nodes.append(ir.Assign(ir.Const(col_name, arr.loc), name, arr.loc)) return self._replace_func( lambda arr, index, name: hpat.hiframes.api.init_series( @@ -405,7 +418,7 @@ def _run_getattr(self, assign, rhs): return [assign] def _handle_df_values(self, lhs, df_var): - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] n_cols = len(df_typ.columns) nodes = [] data_arrs = [self._get_dataframe_data(df_var, c, nodes) @@ -416,7 +429,7 @@ def _handle_df_values(self, lhs, df_var): func_text += " return np.stack(({}), 1)\n".format(data_args) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'np': np}, loc_vars) f = loc_vars['f'] return self._replace_func(f, data_arrs, pre_nodes=nodes) @@ -424,7 +437,7 @@ def _handle_df_values(self, lhs, df_var): def _run_binop(self, assign, rhs): arg1, arg2 = rhs.lhs, rhs.rhs - typ1, typ2 = self.typemap[arg1.name], self.typemap[arg2.name] + typ1, typ2 = self.state.typemap[arg1.name], self.state.typemap[arg2.name] if not (isinstance(typ1, DataFrameType) or isinstance(typ2, DataFrameType)): return [assign] @@ -441,14 +454,14 @@ def _run_binop(self, assign, rhs): # self._convert_series_calltype(rhs) # # output stays as Array in A += B where A is Array - # if isinstance(self.typemap[assign.target.name], types.Array): - # assert isinstance(self.calltypes[rhs].return_type, types.Array) + # if isinstance(self.state.typemap[assign.target.name], types.Array): + # assert isinstance(self.state.calltypes[rhs].return_type, types.Array) # nodes.append(assign) # return nodes # out_data = ir.Var( # arg1.scope, mk_unique_var(assign.target.name+'_data'), rhs.loc) - # self.typemap[out_data.name] = self.calltypes[rhs].return_type + # self.state.typemap[out_data.name] = self.state.calltypes[rhs].return_type # nodes.append(ir.Assign(rhs, out_data, rhs.loc)) # return self._replace_func( # lambda data: hpat.hiframes.api.init_series(data, None, None), @@ -458,7 +471,7 @@ def _run_binop(self, assign, rhs): def _run_unary(self, assign, rhs): arg = rhs.value - typ = self.typemap[arg.name] + typ = self.state.typemap[arg.name] if isinstance(typ, DataFrameType): nodes = [] @@ -467,7 +480,7 @@ def _run_unary(self, assign, rhs): # self._convert_series_calltype(rhs) # out_data = ir.Var( # arg.scope, mk_unique_var(assign.target.name+'_data'), rhs.loc) - # self.typemap[out_data.name] = self.calltypes[rhs].return_type + # self.state.typemap[out_data.name] = self.state.calltypes[rhs].return_type # nodes.append(ir.Assign(rhs, out_data, rhs.loc)) # return self._replace_func( # lambda data: hpat.hiframes.api.init_series(data), @@ -478,11 +491,11 @@ def _run_unary(self, assign, rhs): return [assign] def _run_call(self, assign, lhs, rhs): - fdef = guard(find_callname, self.func_ir, rhs, self.typemap) + fdef = guard(find_callname, self.state.func_ir, rhs, self.state.typemap) if fdef is None: from numba.stencil import StencilFunc # could be make_function from list comprehension which is ok - func_def = guard(get_definition, self.func_ir, rhs.func) + func_def = guard(get_definition, self.state.func_ir, rhs.func) if isinstance(func_def, ir.Expr) and func_def.op == 'make_function': return [assign] if isinstance(func_def, ir.Global) and isinstance(func_def.value, StencilFunc): @@ -500,28 +513,28 @@ def _run_call(self, assign, lhs, rhs): return self._run_call_set_df_column(assign, lhs, rhs) if fdef == ('merge', 'pandas'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.merge_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if fdef == ('merge_asof', 'pandas'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.merge_asof_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if fdef == ('join_dummy', 'hpat.hiframes.api'): return self._run_call_join(assign, lhs, rhs) if (isinstance(func_mod, ir.Var) - and isinstance(self.typemap[func_mod.name], DataFrameType)): + and isinstance(self.state.typemap[func_mod.name], DataFrameType)): return self._run_call_dataframe( assign, assign.target, rhs, func_mod, func_name) @@ -530,13 +543,13 @@ def _run_call(self, assign, lhs, rhs): return [assign] if (isinstance(func_mod, ir.Var) - and isinstance(self.typemap[func_mod.name], + and isinstance(self.state.typemap[func_mod.name], DataFrameGroupByType)): return self._run_call_groupby( assign, assign.target, rhs, func_mod, func_name) if (isinstance(func_mod, ir.Var) - and isinstance(self.typemap[func_mod.name], RollingType)): + and isinstance(self.state.typemap[func_mod.name], RollingType)): return self._run_call_rolling( assign, assign.target, rhs, func_mod, func_name) @@ -544,28 +557,28 @@ def _run_call(self, assign, lhs, rhs): return self._run_call_pivot_table(assign, lhs, rhs) if fdef == ('crosstab', 'pandas'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.crosstab_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if fdef == ('crosstab_dummy', 'hpat.hiframes.pd_groupby_ext'): return self._run_call_crosstab(assign, lhs, rhs) if fdef == ('concat', 'pandas'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.concat_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if (fdef == ('concat_dummy', 'hpat.hiframes.pd_dataframe_ext') - and isinstance(self.typemap[lhs.name], DataFrameType)): + and isinstance(self.state.typemap[lhs.name], DataFrameType)): return self._run_call_concat(assign, lhs, rhs) if fdef == ('sort_values_dummy', 'hpat.hiframes.pd_dataframe_ext'): @@ -593,13 +606,13 @@ def _run_call(self, assign, lhs, rhs): return self._run_call_reset_index(assign, lhs, rhs) if fdef == ('drop_inplace', 'hpat.hiframes.api'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.api.drop_inplace_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if fdef == ('drop_dummy', 'hpat.hiframes.pd_dataframe_ext'): return self._run_call_drop(assign, lhs, rhs) @@ -642,8 +655,8 @@ def _run_call(self, assign, lhs, rhs): def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'merge': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.merge_overload( *arg_typs, **kw_typs) @@ -653,8 +666,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'pivot_table': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.pivot_table_overload( *arg_typs, **kw_typs) @@ -667,8 +680,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'rolling': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_rolling_ext.df_rolling_overload( *arg_typs, **kw_typs) @@ -689,8 +702,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): # df.sort_values() if func_name == 'sort_values': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.sort_values_overload( *arg_typs, **kw_typs) @@ -702,8 +715,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'itertuples': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.itertuples_overload( *arg_typs, **kw_typs) @@ -714,8 +727,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'head': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.head_overload( *arg_typs, **kw_typs) @@ -726,8 +739,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'isna': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.isna_overload( *arg_typs, **kw_typs) @@ -738,8 +751,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'astype': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.astype_overload( *arg_typs, **kw_typs) @@ -750,8 +763,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'fillna': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.fillna_overload( *arg_typs, **kw_typs) @@ -763,8 +776,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'dropna': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.dropna_overload( *arg_typs, **kw_typs) @@ -776,8 +789,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'reset_index': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.reset_index_overload( *arg_typs, **kw_typs) @@ -789,8 +802,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'drop': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.drop_overload( *arg_typs, **kw_typs) @@ -802,8 +815,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'isin': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.isin_overload( *arg_typs, **kw_typs) @@ -814,8 +827,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'append': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.append_overload( *arg_typs, **kw_typs) @@ -827,8 +840,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'pct_change': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.pct_change_overload( *arg_typs, **kw_typs) @@ -840,8 +853,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'mean': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.mean_overload( *arg_typs, **kw_typs) @@ -853,8 +866,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'median': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.median_overload( *arg_typs, **kw_typs) @@ -866,8 +879,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'std': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.std_overload( *arg_typs, **kw_typs) @@ -879,8 +892,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'var': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.var_overload( *arg_typs, **kw_typs) @@ -892,8 +905,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'max': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.max_overload( *arg_typs, **kw_typs) @@ -905,8 +918,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'min': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.min_overload( *arg_typs, **kw_typs) @@ -918,8 +931,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'sum': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.sum_overload( *arg_typs, **kw_typs) @@ -931,8 +944,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'prod': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.prod_overload( *arg_typs, **kw_typs) @@ -944,8 +957,8 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): if func_name == 'count': rhs.args.insert(0, df_var) - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_dataframe_ext.count_overload( *arg_typs, **kw_typs) @@ -957,13 +970,13 @@ def _run_call_dataframe(self, assign, lhs, rhs, df_var, func_name): return [assign] def _run_call_dataframe_apply(self, assign, lhs, rhs, df_var): - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # get apply function kws = dict(rhs.kws) func_var = self._get_arg('apply', rhs.args, kws, 0, 'func') - func = guard(get_definition, self.func_ir, func_var) + func = guard(get_definition, self.state.func_ir, func_var) # TODO: get globals directly from passed lambda if possible? - _globals = self.func_ir.func_id.func.__globals__ + _globals = self.state.func_ir.func_id.func.__globals__ lambda_ir = compile_to_numba_ir(func, _globals) # find columns that are actually used if possible @@ -1014,7 +1027,7 @@ def _run_call_dataframe_apply(self, assign, lhs, rhs, df_var): func_text += " return hpat.hiframes.api.init_series(S)\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'numba': numba, 'Row': Row}, loc_vars) f = loc_vars['f'] f_ir = compile_to_numba_ir( @@ -1032,6 +1045,7 @@ def _run_call_dataframe_apply(self, assign, lhs, rhs, df_var): and stmt.value.op == 'call'): fdef = guard(get_definition, f_ir, stmt.value.func) if isinstance(fdef, ir.Global) and fdef.name == 'map_func': + update_globals(func, _globals) inline_closure_call(f_ir, _globals, block, i, func) # fix the global value to avoid typing errors fdef.value = 1 @@ -1039,10 +1053,10 @@ def _run_call_dataframe_apply(self, assign, lhs, rhs, df_var): break arg_typs = tuple(df_typ.data[df_typ.columns.index(c)] for c in used_cols) - f_typemap, f_return_type, f_calltypes = numba.compiler.type_inference_stage( - self.typingctx, f_ir, arg_typs, None) - self.typemap.update(f_typemap) - self.calltypes.update(f_calltypes) + f_typemap, f_return_type, f_calltypes = numba.typed_passes.type_inference_stage( + self.state.typingctx, f_ir, arg_typs, None) + self.state.typemap.update(f_typemap) + self.state.calltypes.update(f_calltypes) nodes = [] col_vars = [self._get_dataframe_data(df_var, c, nodes) for c in used_cols] @@ -1053,13 +1067,13 @@ def _run_call_dataframe_apply(self, assign, lhs, rhs, df_var): def _handle_df_describe(self, assign, lhs, rhs, df_var): """translate df.describe() call with no input or just include='all' """ - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # check for no arg or just include='all' if not (len(rhs.args) == 0 and (len(rhs.kws) == 0 or (len(rhs.kws) == 1 and rhs.kws[0][0] == 'include' - and guard(find_const, self.func_ir, rhs.kws[0][1]) == 'all'))): + and guard(find_const, self.state.func_ir, rhs.kws[0][1]) == 'all'))): raise ValueError("only describe() with include='all' supported") col_name_args = ["c" + str(i) for i in range(len(df_typ.columns))] @@ -1099,7 +1113,7 @@ def _handle_df_describe(self, assign, lhs, rhs, df_var): func_text += " 'max ' + {}\n".format(max_strs) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np}, loc_vars) f = loc_vars['f'] nodes = [] @@ -1108,9 +1122,9 @@ def _handle_df_describe(self, assign, lhs, rhs, df_var): def _run_call_df_sort_values(self, assign, lhs, rhs): df_var, by_var, ascending_var, inplace_var = rhs.args - df_typ = self.typemap[df_var.name] - ascending = guard(find_const, self.func_ir, ascending_var) - inplace = guard(find_const, self.func_ir, inplace_var) + df_typ = self.state.typemap[df_var.name] + ascending = guard(find_const, self.state.func_ir, ascending_var) + inplace = guard(find_const, self.state.func_ir, inplace_var) # find key array for sort ('by' arg) key_names = self._get_const_or_list(by_var) @@ -1133,7 +1147,7 @@ def _run_call_df_sort_values(self, assign, lhs, rhs): for k in df_typ.columns: out_var = ir.Var(lhs.scope, mk_unique_var(k), lhs.loc) ind = df_typ.columns.index(k) - self.typemap[out_var.name] = df_typ.data[ind] + self.state.typemap[out_var.name] = df_typ.data[ind] out_vars[k] = out_var for k in key_names: out_key_vars.append(out_vars.pop(k)) @@ -1164,7 +1178,7 @@ def _run_call_df_itertuples(self, assign, lhs, rhs): e.g. get_itertuples("A", "B", A_arr, B_arr) """ df_var = rhs.args[0] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] col_name_args = ', '.join(["c" + str(i) for i in range(len(df_typ.columns))]) name_consts = ', '.join(["'{}'".format(c) for c in df_typ.columns]) @@ -1174,7 +1188,7 @@ def _run_call_df_itertuples(self, assign, lhs, rhs): .format(name_consts, col_name_args) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) f = loc_vars['f'] nodes = [] @@ -1184,7 +1198,7 @@ def _run_call_df_itertuples(self, assign, lhs, rhs): def _run_call_df_head(self, assign, lhs, rhs): df_var = rhs.args[0] n = rhs.args[1] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.head(n), get # output data and create a new dataframe @@ -1199,7 +1213,7 @@ def _run_call_df_head(self, assign, lhs, rhs): ", ".join(d + '_O' for d in data_args), ", ".join("'{}'".format(c) for c in df_typ.columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _head_impl = loc_vars['_head_impl'] nodes = [] @@ -1210,7 +1224,7 @@ def _run_call_pct_change(self, assign, lhs, rhs): # TODO: refactor since similar to df.head() and others that call Series df_var = rhs.args[0] periods = rhs.args[1] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.pct_change(periods), get # output data and create a new dataframe @@ -1225,7 +1239,7 @@ def _run_call_pct_change(self, assign, lhs, rhs): ", ".join(d + '_O' for d in data_args), ", ".join("'{}'".format(c) for c in df_typ.columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _pct_change_impl = loc_vars['_pct_change_impl'] nodes = [] @@ -1237,7 +1251,7 @@ def _run_call_col_reduce(self, assign, lhs, rhs, func_name): a Series like mean, std, max, ...""" # TODO: refactor df_var = rhs.args[0] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.mean(), get # output data and create a new indexed Series @@ -1254,7 +1268,7 @@ def _run_call_col_reduce(self, assign, lhs, rhs, func_name): ", ".join("'{}'".format(c) for c in df_typ.columns)) func_text += " return hpat.hiframes.api.init_series(data, index)\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np}, loc_vars) _reduce_impl = loc_vars['_reduce_impl'] nodes = [] @@ -1265,8 +1279,8 @@ def _run_call_df_fillna(self, assign, lhs, rhs): df_var = rhs.args[0] value = rhs.args[1] inplace_var = rhs.args[2] - inplace = guard(find_const, self.func_ir, inplace_var) - df_typ = self.typemap[df_var.name] + inplace = guard(find_const, self.state.func_ir, inplace_var) + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.fillna(), get # output data and create a new dataframe @@ -1285,7 +1299,7 @@ def _run_call_df_fillna(self, assign, lhs, rhs): ", ".join(d + '_O' for d in data_args), ", ".join("'{}'".format(c) for c in df_typ.columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _fillna_impl = loc_vars['_fillna_impl'] nodes = [] @@ -1300,8 +1314,8 @@ def _run_call_df_fillna(self, assign, lhs, rhs): def _run_call_df_dropna(self, assign, lhs, rhs): df_var = rhs.args[0] inplace_var = rhs.args[1] - inplace = guard(find_const, self.func_ir, inplace_var) - df_typ = self.typemap[df_var.name] + inplace = guard(find_const, self.state.func_ir, inplace_var) + df_typ = self.state.typemap[df_var.name] nodes = [] col_vars = [self._get_dataframe_data(df_var, c, nodes) for c in df_typ.columns] @@ -1314,15 +1328,15 @@ def _run_call_df_dropna(self, assign, lhs, rhs): func_text += " ({},) = hpat.hiframes.api.dropna(({},), inplace)\n".format( out_names, arg_names) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _dropna_imp = loc_vars['_dropna_imp'] f_block = compile_to_numba_ir(_dropna_imp, {'hpat': hpat}, - self.typingctx, - df_typ.data + (self.typemap[inplace_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + df_typ.data + (self.state.typemap[inplace_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, col_vars + [inplace_var]) nodes += f_block.body[:-3] @@ -1345,8 +1359,8 @@ def _run_call_reset_index(self, assign, lhs, rhs): # TODO: drop actual index, fix inplace df_var = rhs.args[0] inplace_var = rhs.args[1] - inplace = guard(find_const, self.func_ir, inplace_var) - df_typ = self.typemap[df_var.name] + inplace = guard(find_const, self.state.func_ir, inplace_var) + df_typ = self.state.typemap[df_var.name] # impl: for each column, copy data and create a new dataframe n_cols = len(df_typ.columns) @@ -1360,7 +1374,7 @@ def _run_call_reset_index(self, assign, lhs, rhs): ", ".join(data_args), ", ".join("'{}'".format(c) for c in df_typ.columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _reset_index_impl = loc_vars['_reset_index_impl'] nodes = [] @@ -1374,9 +1388,9 @@ def _run_call_reset_index(self, assign, lhs, rhs): def _run_call_drop(self, assign, lhs, rhs): df_var, labels_var, axis_var, columns_var, inplace_var = rhs.args - inplace = guard(find_const, self.func_ir, inplace_var) - df_typ = self.typemap[df_var.name] - out_typ = self.typemap[lhs.name] + inplace = guard(find_const, self.state.func_ir, inplace_var) + df_typ = self.state.typemap[df_var.name] + out_typ = self.state.typemap[lhs.name] # TODO: reflection for drop inplace nodes = [] @@ -1389,7 +1403,7 @@ def _run_call_drop(self, assign, lhs, rhs): def _run_call_df_isna(self, assign, lhs, rhs): df_var = rhs.args[0] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.isna(), get # output data and create a new dataframe @@ -1410,7 +1424,7 @@ def _run_call_df_isna(self, assign, lhs, rhs): func_text = '\n'.join(func_lines) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _isna_impl = loc_vars['_isna_impl'] nodes = [] @@ -1424,7 +1438,7 @@ def _run_call_df_astype(self, assign, lhs, rhs): ''' df_var = rhs.args[0] dtype_var = rhs.args[1] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # impl: for each column, convert data to series, call S.astype(dtype_value), get # output data and create a new dataframe @@ -1456,8 +1470,8 @@ def _run_call_df_astype(self, assign, lhs, rhs): def _run_call_isin(self, assign, lhs, rhs): df_var, values = rhs.args - df_typ = self.typemap[df_var.name] - values_typ = self.typemap[values.name] + df_typ = self.state.typemap[df_var.name] + values_typ = self.state.typemap[values.name] nodes = [] other_colmap = {} @@ -1502,10 +1516,10 @@ def bool_arr_func(A): f_block = compile_to_numba_ir( func, {'hpat': hpat, 'np': np}, - self.typingctx, - tuple(self.typemap[v.name] for v in args), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + tuple(self.state.typemap[v.name] for v in args), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, args) nodes += f_block.body[:-2] out_vars.append(nodes[-1].target) @@ -1523,27 +1537,27 @@ def _impl(target, index, val): return self._replace_func(_impl, rhs.args) df_var = rhs.args[0] - cname = guard(find_const, self.func_ir, rhs.args[1]) + cname = guard(find_const, self.state.func_ir, rhs.args[1]) new_arr = rhs.args[2] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] nodes = [] # find df['col2'] = df['col1'][arr] # since columns should have the same size, output is filled with NaNs # TODO: make sure col1 and col2 are in the same df - arr_def = guard(get_definition, self.func_ir, new_arr) + arr_def = guard(get_definition, self.state.func_ir, new_arr) if (isinstance(arr_def, ir.Expr) and arr_def.op == 'getitem' - and is_array(self.typemap, arr_def.value.name) + and is_array(self.state.typemap, arr_def.value.name) and self.is_bool_arr(arr_def.index.name)): orig_arr = arr_def.value bool_arr = arr_def.index f_block = compile_to_numba_ir( lambda arr, bool_arr: hpat.hiframes.api.series_filter_bool(arr, bool_arr), {'hpat': hpat}, - self.typingctx, - (self.typemap[orig_arr.name], self.typemap[bool_arr.name]), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[orig_arr.name], self.state.typemap[bool_arr.name]), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [orig_arr, bool_arr]) nodes += f_block.body[:-2] @@ -1578,12 +1592,12 @@ def _impl(target, index, val): func_text += " return hpat.hiframes.pd_dataframe_ext.init_dataframe({}, None, {})\n".format( data_args, col_args) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _init_df = loc_vars['_init_df'] return self._replace_func(_init_df, in_arrs, pre_nodes=nodes) def _run_call_len(self, lhs, df_var): - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] # empty dataframe has 0 len if len(df_typ.columns) == 0: @@ -1604,8 +1618,8 @@ def _run_call_join(self, assign, lhs, rhs): left_on = self._get_const_or_list(left_on_var) right_on = self._get_const_or_list(right_on_var) - how = guard(find_const, self.func_ir, how_var) - out_typ = self.typemap[lhs.name] + how = guard(find_const, self.state.func_ir, how_var) + out_typ = self.state.typemap[lhs.name] # convert right join to left join if how == 'right': @@ -1617,13 +1631,13 @@ def _run_call_join(self, assign, lhs, rhs): out_data_vars = {c: ir.Var(lhs.scope, mk_unique_var(c), lhs.loc) for c in out_typ.columns} for v, t in zip(out_data_vars.values(), out_typ.data): - self.typemap[v.name] = t + self.state.typemap[v.name] = t left_arrs = {c: self._get_dataframe_data(left_df, c, nodes) - for c in self.typemap[left_df.name].columns} + for c in self.state.typemap[left_df.name].columns} right_arrs = {c: self._get_dataframe_data(right_df, c, nodes) - for c in self.typemap[right_df.name].columns} + for c in self.state.typemap[right_df.name].columns} nodes.append(hiframes.join.Join(lhs.name, left_df.name, right_df.name, @@ -1636,10 +1650,10 @@ def _run_call_join(self, assign, lhs, rhs): pre_nodes=nodes) def _run_call_groupby(self, assign, lhs, rhs, grp_var, func_name): - grp_typ = self.typemap[grp_var.name] + grp_typ = self.state.typemap[grp_var.name] df_var = self._get_df_obj_select(grp_var, 'groupby') - df_type = self.typemap[df_var.name] - out_typ = self.typemap[lhs.name] + df_type = self.state.typemap[df_var.name] + out_typ = self.state.typemap[lhs.name] nodes = [] in_vars = {c: self._get_dataframe_data(df_var, c, nodes) @@ -1654,18 +1668,18 @@ def _run_call_groupby(self, assign, lhs, rhs, grp_var, func_name): for k in grp_typ.keys: out_key_var = ir.Var(lhs.scope, mk_unique_var(k), lhs.loc) ind = out_typ.columns.index(k) - self.typemap[out_key_var.name] = out_typ.data[ind] + self.state.typemap[out_key_var.name] = out_typ.data[ind] out_key_vars.append(out_key_var) df_col_map = {} for c in grp_typ.selection: var = ir.Var(lhs.scope, mk_unique_var(c), lhs.loc) - self.typemap[var.name] = (out_typ.data + self.state.typemap[var.name] = (out_typ.data if isinstance(out_typ, SeriesType) else out_typ.data[out_typ.columns.index(c)]) df_col_map[c] = var - agg_func = get_agg_func(self.func_ir, func_name, rhs) + agg_func = get_agg_func(self.state.func_ir, func_name, rhs) agg_node = hiframes.aggregate.Aggregate( lhs.name, df_var.name, grp_typ.keys, out_key_vars, df_col_map, @@ -1700,13 +1714,13 @@ def _run_call_groupby(self, assign, lhs, rhs, grp_var, func_name): def _run_call_pivot_table(self, assign, lhs, rhs): df_var, values, index, columns, aggfunc, _pivot_values = rhs.args - func_name = self.typemap[aggfunc.name].literal_value - values = self.typemap[values.name].literal_value - index = self.typemap[index.name].literal_value - columns = self.typemap[columns.name].literal_value - pivot_values = self.typemap[_pivot_values.name].meta - df_type = self.typemap[df_var.name] - out_typ = self.typemap[lhs.name] + func_name = self.state.typemap[aggfunc.name].literal_value + values = self.state.typemap[values.name].literal_value + index = self.state.typemap[index.name].literal_value + columns = self.state.typemap[columns.name].literal_value + pivot_values = self.state.typemap[_pivot_values.name].meta + df_type = self.state.typemap[df_var.name] + out_typ = self.state.typemap[lhs.name] nodes = [] in_vars = {values: self._get_dataframe_data(df_var, values, nodes)} @@ -1714,11 +1728,11 @@ def _run_call_pivot_table(self, assign, lhs, rhs): df_col_map = ({col: ir.Var(lhs.scope, mk_unique_var(col), lhs.loc) for col in pivot_values}) for v in df_col_map.values(): - self.typemap[v.name] = out_typ.data[0] + self.state.typemap[v.name] = out_typ.data[0] pivot_arr = self._get_dataframe_data(df_var, columns, nodes) index_arr = self._get_dataframe_data(df_var, index, nodes) - agg_func = get_agg_func(self.func_ir, func_name, rhs) + agg_func = get_agg_func(self.state.func_ir, func_name, rhs) agg_node = hiframes.aggregate.Aggregate( lhs.name, df_var.name, [index], None, df_col_map, @@ -1738,15 +1752,15 @@ def _run_call_pivot_table(self, assign, lhs, rhs): def _run_call_crosstab(self, assign, lhs, rhs): index, columns, _pivot_values = rhs.args - pivot_values = self.typemap[_pivot_values.name].meta - out_typ = self.typemap[lhs.name] + pivot_values = self.state.typemap[_pivot_values.name].meta + out_typ = self.state.typemap[lhs.name] in_vars = {} df_col_map = ({col: ir.Var(lhs.scope, mk_unique_var(col), lhs.loc) for col in pivot_values}) for i, v in enumerate(df_col_map.values()): - self.typemap[v.name] = out_typ.data[i] + self.state.typemap[v.name] = out_typ.data[i] pivot_arr = columns @@ -1776,31 +1790,31 @@ def _agg_len_impl(in_arr): # pragma: no cover pre_nodes=nodes) def _run_call_rolling(self, assign, lhs, rhs, rolling_var, func_name): - rolling_typ = self.typemap[rolling_var.name] - dummy_call = guard(get_definition, self.func_ir, rolling_var) + rolling_typ = self.state.typemap[rolling_var.name] + dummy_call = guard(get_definition, self.state.func_ir, rolling_var) df_var, window, center, on = dummy_call.args - df_type = self.typemap[df_var.name] - out_typ = self.typemap[lhs.name] + df_type = self.state.typemap[df_var.name] + out_typ = self.state.typemap[lhs.name] # handle 'on' arg - if self.typemap[on.name] == types.none: + if self.state.typemap[on.name] == types.none: on = None else: - assert isinstance(self.typemap[on.name], types.StringLiteral) - on = self.typemap[on.name].literal_value + assert isinstance(self.state.typemap[on.name], types.StringLiteral) + on = self.state.typemap[on.name].literal_value nodes = [] # convert string offset window statically to nanos # TODO: support dynamic conversion # TODO: support other offsets types (time delta, etc.) if on is not None: - window = guard(find_const, self.func_ir, window) + window = guard(find_const, self.state.func_ir, window) if not isinstance(window, str): raise ValueError("window argument to rolling should be constant" "string in the offset case (variable window)") window = pd.tseries.frequencies.to_offset(window).nanos window_var = ir.Var(lhs.scope, mk_unique_var("window"), lhs.loc) - self.typemap[window_var.name] = types.int64 + self.state.typemap[window_var.name] = types.int64 nodes.append(ir.Assign(ir.Const(window, lhs.loc), window_var, lhs.loc)) window = window_var @@ -1813,7 +1827,7 @@ def _run_call_rolling(self, assign, lhs, rhs, rolling_var, func_name): df_col_map = {} for c in rolling_typ.selection: var = ir.Var(lhs.scope, mk_unique_var(c), lhs.loc) - self.typemap[var.name] = (out_typ.data + self.state.typemap[var.name] = (out_typ.data if isinstance(out_typ, SeriesType) else out_typ.data[out_typ.columns.index(c)]) df_col_map[c] = var @@ -1831,7 +1845,7 @@ def _run_call_rolling(self, assign, lhs, rhs, rolling_var, func_name): in_col_var = in_vars[cname] if func_name in ('cov', 'corr'): # TODO: Series as other - if cname not in self.typemap[other.name].columns: + if cname not in self.state.typemap[other.name].columns: continue # nan column handled below rhs.args[0] = self._get_dataframe_data(other, cname, nodes) nodes += self._gen_rolling_call( @@ -1840,17 +1854,17 @@ def _run_call_rolling(self, assign, lhs, rhs, rolling_var, func_name): # in corr/cov case, Pandas makes non-common columns NaNs if func_name in ('cov', 'corr'): - nan_cols = list(set(self.typemap[other.name].columns) ^ set(df_type.columns)) + nan_cols = list(set(self.state.typemap[other.name].columns) ^ set(df_type.columns)) len_arr = list(in_vars.values())[0] for cname in nan_cols: def f(arr): nan_arr = np.full(len(arr), np.nan) f_block = compile_to_numba_ir(f, {'np': np}, - self.typingctx, - (self.typemap[len_arr.name],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[len_arr.name],), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [len_arr]) nodes += f_block.body[:-3] # remove none return df_col_map[cname] = nodes[-1].target @@ -1924,13 +1938,13 @@ def f(arr, w, center): # pragma: no cover arr, w, center, False, _func_name) args = [in_col_var, window, center] - arg_typs = tuple(self.typemap[v.name] for v in args) + arg_typs = tuple(self.state.typemap[v.name] for v in args) f_block = compile_to_numba_ir(f, {'hpat': hpat, '_func_name': func_name}, - self.typingctx, + self.state.typingctx, arg_typs, - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, args) nodes += f_block.body[:-3] # remove none return nodes[-1].target = out_col_var @@ -1939,9 +1953,9 @@ def f(arr, w, center): # pragma: no cover def _run_call_concat(self, assign, lhs, rhs): # TODO: handle non-numerical (e.g. string, datetime) columns nodes = [] - out_typ = self.typemap[lhs.name] + out_typ = self.state.typemap[lhs.name] df_list = self._get_const_tup(rhs.args[0]) - axis = guard(find_const, self.func_ir, rhs.args[1]) + axis = guard(find_const, self.state.func_ir, rhs.args[1]) if axis == 1: return self._run_call_concat_columns(df_list, out_typ) @@ -1955,7 +1969,7 @@ def gen_nan_func(A): func_text += " return hpat.hiframes.api.concat(({}))\n".format( arg_names) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _concat_imp = loc_vars['_concat_imp'] out_vars = [] @@ -1964,15 +1978,15 @@ def gen_nan_func(A): args = [] # get input columns for df in df_list: - df_typ = self.typemap[df.name] + df_typ = self.state.typemap[df.name] # generate full NaN column if cname not in df_typ.columns: f_block = compile_to_numba_ir(gen_nan_func, {'hpat': hpat, 'np': np}, - self.typingctx, + self.state.typingctx, (df_typ.data[0],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] arr = self._get_dataframe_data(df, df_typ.columns[0], nodes) replace_arg_nodes(f_block, [arr]) nodes += f_block.body[:-2] @@ -1981,13 +1995,13 @@ def gen_nan_func(A): arr = self._get_dataframe_data(df, cname, nodes) args.append(arr) - arg_typs = tuple(self.typemap[v.name] for v in args) + arg_typs = tuple(self.state.typemap[v.name] for v in args) f_block = compile_to_numba_ir(_concat_imp, {'hpat': hpat, 'np': np}, - self.typingctx, + self.state.typingctx, arg_typs, - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, args) nodes += f_block.body[:-2] out_vars.append(nodes[-1].target) @@ -2004,10 +2018,10 @@ def _run_call_concat_columns(self, objs, out_typ): f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_series_data(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[obj.name],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[obj.name],), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, (obj,)) nodes += f_block.body[:-2] out_vars.append(nodes[-1].target) @@ -2021,13 +2035,13 @@ def _get_df_obj_select(self, obj_var, obj_name): """get df object for groupby() or rolling() e.g. groupby('A')['B'], groupby('A')['B', 'C'], groupby('A') """ - select_def = guard(get_definition, self.func_ir, obj_var) + select_def = guard(get_definition, self.state.func_ir, obj_var) if isinstance(select_def, ir.Expr) and select_def.op in ('getitem', 'static_getitem'): obj_var = select_def.value - obj_call = guard(get_definition, self.func_ir, obj_var) + obj_call = guard(get_definition, self.state.func_ir, obj_var) # find dataframe - call_def = guard(find_callname, self.func_ir, obj_call) + call_def = guard(find_callname, self.state.func_ir, obj_call) assert (call_def is not None and call_def[0] == obj_name and isinstance(call_def[1], ir.Var) and self._is_df_var(call_def[1])) @@ -2036,7 +2050,7 @@ def _get_df_obj_select(self, obj_var, obj_name): return df_var def _get_const_tup(self, tup_var): - tup_def = guard(get_definition, self.func_ir, tup_var) + tup_def = guard(get_definition, self.state.func_ir, tup_var) if isinstance(tup_def, ir.Expr): if tup_def.op == 'binop' and tup_def.fn in ('+', operator.add): return (self._get_const_tup(tup_def.lhs) @@ -2050,14 +2064,14 @@ def _set_df_inplace(self, _init_df, out_arrs, df_var, loc, nodes): # also, boxed dfs need to be updated # HACK assign output df back to input df variables # TODO CFG backbone? - df_typ = self.typemap[df_var.name] - arg_typs = tuple(self.typemap[v.name] for v in out_arrs) + df_typ = self.state.typemap[df_var.name] + arg_typs = tuple(self.state.typemap[v.name] for v in out_arrs) f_block = compile_to_numba_ir(_init_df, {'hpat': hpat}, - self.typingctx, + self.state.typingctx, arg_typs, - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, out_arrs) nodes += f_block.body[:-2] new_df = nodes[-1].target @@ -2067,15 +2081,15 @@ def _set_df_inplace(self, _init_df, out_arrs, df_var, loc, nodes): f_block = compile_to_numba_ir( lambda df: hpat.hiframes.pd_dataframe_ext.set_parent_dummy(df), {'hpat': hpat}, - self.typingctx, - (self.typemap[new_df.name],), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[new_df.name],), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [new_df]) nodes += f_block.body[:-2] new_df = nodes[-1].target - for other_df_var in self.func_ir._definitions[df_var.name]: + for other_df_var in self.state.func_ir._definitions[df_var.name]: if isinstance(other_df_var, ir.Var): nodes.append(ir.Assign(new_df, other_df_var, loc)) ir.Assign(new_df, df_var, loc) @@ -2088,16 +2102,16 @@ def _get_dataframe_data(self, df_var, col_name, nodes): # e.g. A = init_dataframe(A, None, 'A') # XXX assuming init_dataframe is the only call to create a dataframe # and dataframe._data is never overwritten - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] ind = df_typ.columns.index(col_name) - var_def = guard(get_definition, self.func_ir, df_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, df_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_dataframe', 'hpat.hiframes.pd_dataframe_ext'): return var_def.args[ind] loc = df_var.loc ind_var = ir.Var(df_var.scope, mk_unique_var('col_ind'), loc) - self.typemap[ind_var.name] = types.IntegerLiteral(ind) + self.state.typemap[ind_var.name] = types.IntegerLiteral(ind) nodes.append(ir.Assign(ir.Const(ind, loc), ind_var, loc)) # XXX use get_series_data() for getting data instead of S._data # to enable alias analysis @@ -2105,20 +2119,20 @@ def _get_dataframe_data(self, df_var, col_name, nodes): lambda df, c_ind: hpat.hiframes.pd_dataframe_ext.get_dataframe_data( df, c_ind), {'hpat': hpat}, - self.typingctx, - (df_typ, self.typemap[ind_var.name]), - self.typemap, - self.calltypes + self.state.typingctx, + (df_typ, self.state.typemap[ind_var.name]), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [df_var, ind_var]) nodes += f_block.body[:-2] return nodes[-1].target def _get_dataframe_index(self, df_var, nodes): - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] n_cols = len(df_typ.columns) - var_def = guard(get_definition, self.func_ir, df_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, df_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_dataframe', 'hpat.hiframes.pd_dataframe_ext'): return var_def.args[n_cols] @@ -2127,10 +2141,10 @@ def _get_dataframe_index(self, df_var, nodes): f_block = compile_to_numba_ir( lambda df: hpat.hiframes.pd_dataframe_ext.get_dataframe_index(df), {'hpat': hpat}, - self.typingctx, + self.state.typingctx, (df_typ,), - self.typemap, - self.calltypes + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [df_var]) nodes += f_block.body[:-2] @@ -2146,7 +2160,7 @@ def _replace_func(self, func, args, const=False, # XXX: inine_closure_call() can't handle defaults properly if pysig is not None: pre_nodes = [] if pre_nodes is None else pre_nodes - scope = next(iter(self.func_ir.blocks.values())).scope + scope = next(iter(self.state.func_ir.blocks.values())).scope loc = scope.loc def normal_handler(index, param, default): @@ -2154,7 +2168,7 @@ def normal_handler(index, param, default): def default_handler(index, param, default): d_var = ir.Var(scope, mk_unique_var('defaults'), loc) - self.typemap[d_var.name] = numba.typeof(default) + self.state.typemap[d_var.name] = numba.typeof(default) node = ir.Assign(ir.Const(default, loc), d_var, loc) pre_nodes.append(node) return d_var @@ -2164,12 +2178,12 @@ def default_handler(index, param, default): pysig, args, kws, normal_handler, default_handler, normal_handler) - arg_typs = tuple(self.typemap[v.name] for v in args) + arg_typs = tuple(self.state.typemap[v.name] for v in args) if const: new_args = [] for i, arg in enumerate(args): - val = guard(find_const, self.func_ir, arg) + val = guard(find_const, self.state.func_ir, arg) if val: new_args.append(types.literal(val)) else: @@ -2178,36 +2192,36 @@ def default_handler(index, param, default): return ReplaceFunc(func, arg_typs, args, glbls, pre_nodes) def _is_df_var(self, var): - return isinstance(self.typemap[var.name], DataFrameType) + return isinstance(self.state.typemap[var.name], DataFrameType) def _is_df_loc_var(self, var): - return isinstance(self.typemap[var.name], DataFrameLocType) + return isinstance(self.state.typemap[var.name], DataFrameLocType) def _is_df_iloc_var(self, var): - return isinstance(self.typemap[var.name], DataFrameILocType) + return isinstance(self.state.typemap[var.name], DataFrameILocType) def _is_df_iat_var(self, var): - return isinstance(self.typemap[var.name], DataFrameIatType) + return isinstance(self.state.typemap[var.name], DataFrameIatType) def is_bool_arr(self, varname): - typ = self.typemap[varname] + typ = self.state.typemap[varname] return (isinstance(typ, (SeriesType, types.Array)) and typ.dtype == types.bool_) def is_int_list_or_arr(self, varname): - typ = self.typemap[varname] + typ = self.state.typemap[varname] return (isinstance(typ, (SeriesType, types.Array, types.List)) and isinstance(typ.dtype, types.Integer)) def _is_const_none(self, var): - var_def = guard(get_definition, self.func_ir, var) + var_def = guard(get_definition, self.state.func_ir, var) return isinstance(var_def, ir.Const) and var_def.value is None def _update_definitions(self, node_list): loc = ir.Loc("", 0) dumm_block = ir.Block(ir.Scope(None, loc), loc) dumm_block.body = node_list - build_definitions({0: dumm_block}, self.func_ir._definitions) + build_definitions({0: dumm_block}, self.state.func_ir._definitions) return def _get_arg(self, f_name, args, kws, arg_no, arg_name, default=None, @@ -2228,25 +2242,25 @@ def _get_arg(self, f_name, args, kws, arg_no, arg_name, default=None, def _gen_arr_copy(self, in_arr, nodes): f_block = compile_to_numba_ir( - lambda A: A.copy(), {}, self.typingctx, - (self.typemap[in_arr.name],), self.typemap, self.calltypes + lambda A: A.copy(), {}, self.state.typingctx, + (self.state.typemap[in_arr.name],), self.state.typemap, self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [in_arr]) nodes += f_block.body[:-2] return nodes[-1].target def _get_const_or_list(self, by_arg, list_only=False, default=None, err_msg=None, typ=None): - var_typ = self.typemap[by_arg.name] + var_typ = self.state.typemap[by_arg.name] if isinstance(var_typ, types.Optional): var_typ = var_typ.type if hasattr(var_typ, 'consts'): return var_typ.consts typ = str if typ is None else typ - by_arg_def = guard(find_build_sequence, self.func_ir, by_arg) + by_arg_def = guard(find_build_sequence, self.state.func_ir, by_arg) if by_arg_def is None: # try single key column - by_arg_def = guard(find_const, self.func_ir, by_arg) + by_arg_def = guard(find_const, self.state.func_ir, by_arg) if by_arg_def is None: if default is not None: return default @@ -2260,7 +2274,7 @@ def _get_const_or_list(self, by_arg, list_only=False, default=None, err_msg=None if default is not None: return default raise ValueError(err_msg) - key_colnames = tuple(guard(find_const, self.func_ir, v) for v in by_arg_def[0]) + key_colnames = tuple(guard(find_const, self.state.func_ir, v) for v in by_arg_def[0]) if any(not isinstance(v, typ) for v in key_colnames): if default is not None: return default @@ -2276,7 +2290,7 @@ def _gen_init_df(columns): func_text += " return hpat.hiframes.pd_dataframe_ext.init_dataframe({}, None, {})\n".format( data_args, ", ".join("'{}'".format(c) for c in columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _init_df = loc_vars['_init_df'] return _init_df diff --git a/hpat/hiframes/hiframes_typed.py b/hpat/hiframes/hiframes_typed.py index db3284ae9..e7b826809 100644 --- a/hpat/hiframes/hiframes_typed.py +++ b/hpat/hiframes/hiframes_typed.py @@ -17,10 +17,11 @@ from numba.inline_closurecall import inline_closure_call from numba.typing.arraydecl import ArrayAttribute from numba.typing.templates import Signature, bound_function, signature, infer_global, AbstractTemplate, signature +from numba.compiler_machinery import FunctionPass, register_pass import hpat from hpat.utils import (debug_prints, inline_new_blocks, ReplaceFunc, - is_whole_slice, is_array) + is_whole_slice, is_array, update_globals) from hpat.str_ext import (string_type, unicode_to_std_str, std_str_to_unicode, list_string_array_type) from hpat.str_arr_ext import (string_array_type, StringArrayType, @@ -79,20 +80,28 @@ '**': '**', } - -class HiFramesTyped(object): +@register_pass(mutates_CFG=True, analysis_only=False) +class HiFramesTypedPass(FunctionPass): """Analyze and transform hiframes calls after typing""" - def __init__(self, func_ir, typingctx, typemap, calltypes): - self.func_ir = func_ir - self.typingctx = typingctx - self.typemap = typemap - self.calltypes = calltypes + _name = "hi_frames_typed_pass" + + def __init__(self): + pass + + def run_pass(self, state): + return HiFramesTypedPassImpl(state).run_pass() + + +class HiFramesTypedPassImpl(object): + + def __init__(self, state): # keep track of tuple variables change by to_const_tuple self._type_changed_vars = [] + self.state = state - def run(self): - blocks = self.func_ir.blocks + def run_pass(self): + blocks = self.state.func_ir.blocks # topo_order necessary so Series data replacement optimization can be # performed in one pass topo_order = find_topo_order(blocks) @@ -105,7 +114,7 @@ def run(self): out_nodes = [inst] if isinstance(inst, ir.Assign): - self.func_ir._definitions[inst.target.name].remove(inst.value) + self.state.func_ir._definitions[inst.target.name].remove(inst.value) out_nodes = self._run_assign(inst) elif isinstance(inst, (ir.SetItem, ir.StaticSetItem)): out_nodes = self._run_setitem(inst) @@ -129,15 +138,16 @@ def run(self): ir.Var(block.scope, "dummy", inst.loc), rp_func.args, (), inst.loc) block.body = new_body + block.body[i:] - inline_closure_call(self.func_ir, rp_func.glbls, - block, len(new_body), rp_func.func, self.typingctx, + update_globals(rp_func.func, rp_func.glbls) + inline_closure_call(self.state.func_ir, rp_func.glbls, + block, len(new_body), rp_func.func, self.state.typingctx, rp_func.arg_types, - self.typemap, self.calltypes, work_list) + self.state.typemap, self.state.calltypes, work_list) replaced = True break if isinstance(out_nodes, dict): block.body = new_body + block.body[i:] - inline_new_blocks(self.func_ir, block, i, out_nodes, work_list) + inline_new_blocks(self.state.func_ir, block, i, out_nodes, work_list) replaced = True break @@ -145,14 +155,15 @@ def run(self): blocks[label].body = new_body # XXX remove slice() of h5 read due to Numba's #3380 bug - self.func_ir.blocks = ir_utils.simplify_CFG(self.func_ir.blocks) - while ir_utils.remove_dead(self.func_ir.blocks, self.func_ir.arg_names, - self.func_ir, self.typemap): + self.state.func_ir.blocks = ir_utils.simplify_CFG(self.state.func_ir.blocks) + while ir_utils.remove_dead(self.state.func_ir.blocks, self.state.func_ir.arg_names, + self.state.func_ir, self.state.typemap): pass - self.func_ir._definitions = build_definitions(self.func_ir.blocks) - dprint_func_ir(self.func_ir, "after hiframes_typed") - return + self.state.func_ir._definitions = build_definitions(self.state.func_ir.blocks) + dprint_func_ir(self.state.func_ir, "after hiframes_typed") + + return True def _run_assign(self, assign): lhs = assign.target.name @@ -160,8 +171,8 @@ def _run_assign(self, assign): # fix type of lhs if type of rhs has been changed if isinstance(rhs, ir.Var) and rhs.name in self._type_changed_vars: - self.typemap.pop(lhs) - self.typemap[lhs] = self.typemap[rhs.name] + self.state.typemap.pop(lhs) + self.state.typemap[lhs] = self.state.typemap[rhs.name] self._type_changed_vars.append(lhs) if isinstance(rhs, ir.Expr): @@ -194,18 +205,18 @@ def _run_getitem(self, assign, rhs): nodes = [] # Series(bool) as index if (rhs.op == 'getitem' - and self.typemap[rhs.index.name] == SeriesType(types.bool_)): + and self.state.typemap[rhs.index.name] == SeriesType(types.bool_)): rhs.index = self._get_series_data(rhs.index, nodes) - if isinstance(self.typemap[rhs.value.name], SeriesIatType): - val_def = guard(get_definition, self.func_ir, rhs.value) + if isinstance(self.state.typemap[rhs.value.name], SeriesIatType): + val_def = guard(get_definition, self.state.func_ir, rhs.value) assert (isinstance(val_def, ir.Expr) and val_def.op == 'getattr' and val_def.attr in ('iat', 'iloc', 'loc')) series_var = val_def.value rhs.value = series_var # replace getitems on dt_index/dt64 series with Timestamp function - if is_dt64_series_typ(self.typemap[rhs.value.name]): + if is_dt64_series_typ(self.state.typemap[rhs.value.name]): if rhs.op == 'getitem': ind_var = rhs.index else: @@ -219,27 +230,27 @@ def f(_in_arr, _ind): return hpat.hiframes.pd_timestamp_ext.convert_datetime64_to_timestamp(s) data = self._get_series_data(in_arr, nodes) - assert isinstance(self.typemap[ind_var.name], + assert isinstance(self.state.typemap[ind_var.name], (types.Integer, types.IntegerLiteral)) f_block = compile_to_numba_ir(f, {'numba': numba, 'np': np, - 'hpat': hpat}, self.typingctx, - (self.typemap[data.name], types.intp), - self.typemap, self.calltypes).blocks.popitem()[1] + 'hpat': hpat}, self.state.typingctx, + (self.state.typemap[data.name], types.intp), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [data, ind_var]) nodes += f_block.body[:-2] nodes[-1].target = assign.target return nodes - if isinstance(self.typemap[rhs.value.name], SeriesType): + if isinstance(self.state.typemap[rhs.value.name], SeriesType): rhs.value = self._get_series_data(rhs.value, nodes) self._convert_series_calltype(rhs) lhs = assign.target # convert output to Series from Array - if isinstance(self.typemap[lhs.name], SeriesType): + if isinstance(self.state.typemap[lhs.name], SeriesType): new_lhs = ir.Var( lhs.scope, mk_unique_var(lhs.name + '_data'), lhs.loc) - self.typemap[new_lhs.name] = series_to_array_type( - self.typemap[lhs.name]) + self.state.typemap[new_lhs.name] = series_to_array_type( + self.state.typemap[lhs.name]) nodes.append(ir.Assign(rhs, new_lhs, lhs.loc)) return self._replace_func( lambda A: hpat.hiframes.api.init_series(A), [new_lhs], @@ -252,23 +263,23 @@ def f(_in_arr, _ind): return nodes def _run_setitem(self, inst): - target_typ = self.typemap[inst.target.name] + target_typ = self.state.typemap[inst.target.name] # Series as index # TODO: handle all possible cases nodes = [] if (isinstance(inst, ir.SetItem) - and isinstance(self.typemap[inst.index.name], SeriesType)): + and isinstance(self.state.typemap[inst.index.name], SeriesType)): inst.index = self._get_series_data(inst.index, nodes) if (isinstance(inst, ir.SetItem) - and isinstance(self.typemap[inst.value.name], SeriesType)): + and isinstance(self.state.typemap[inst.value.name], SeriesType)): inst.value = self._get_series_data(inst.value, nodes) if target_typ == h5dataset_type: return self._handle_h5_write(inst.target, inst.index, inst.value) if isinstance(target_typ, SeriesIatType): - val_def = guard(get_definition, self.func_ir, inst.target) + val_def = guard(get_definition, self.state.func_ir, inst.target) assert (isinstance(val_def, ir.Expr) and val_def.op == 'getattr' and val_def.attr in ('iat', 'iloc', 'loc')) series_var = val_def.value @@ -287,7 +298,7 @@ def _run_setitem(self, inst): return nodes def _run_getattr(self, assign, rhs): - rhs_type = self.typemap[rhs.value.name] # get type of rhs value "S" + rhs_type = self.state.typemap[rhs.value.name] # get type of rhs value "S" # replace arr.dtype for dt64 since PA replaces with # np.datetime64[ns] which invalid, TODO: fix PA @@ -343,7 +354,7 @@ def _run_getattr(self, assign, rhs): return self._run_DatetimeIndex_date(assign, assign.target, rhs) if rhs_type == series_dt_methods_type: - dt_def = guard(get_definition, self.func_ir, rhs.value) + dt_def = guard(get_definition, self.state.func_ir, rhs.value) if dt_def is None: # TODO: check for errors raise ValueError("invalid series.dt") rhs.value = dt_def.value @@ -375,7 +386,7 @@ def _run_binop(self, assign, rhs): return self._handle_dt_index_binop(assign, rhs) arg1, arg2 = rhs.lhs, rhs.rhs - typ1, typ2 = self.typemap[arg1.name], self.typemap[arg2.name] + typ1, typ2 = self.state.typemap[arg1.name], self.state.typemap[arg2.name] if not (isinstance(typ1, SeriesType) or isinstance(typ2, SeriesType)): return [assign] @@ -392,14 +403,14 @@ def _run_binop(self, assign, rhs): self._convert_series_calltype(rhs) # output stays as Array in A += B where A is Array - if isinstance(self.typemap[assign.target.name], types.Array): - assert isinstance(self.calltypes[rhs].return_type, types.Array) + if isinstance(self.state.typemap[assign.target.name], types.Array): + assert isinstance(self.state.calltypes[rhs].return_type, types.Array) nodes.append(assign) return nodes out_data = ir.Var( arg1.scope, mk_unique_var(assign.target.name + '_data'), rhs.loc) - self.typemap[out_data.name] = self.calltypes[rhs].return_type + self.state.typemap[out_data.name] = self.state.calltypes[rhs].return_type nodes.append(ir.Assign(rhs, out_data, rhs.loc)) return self._replace_func( lambda data: hpat.hiframes.api.init_series(data, None, None), @@ -409,7 +420,7 @@ def _run_binop(self, assign, rhs): def _run_unary(self, assign, rhs): arg = rhs.value - typ = self.typemap[arg.name] + typ = self.state.typemap[arg.name] if isinstance(typ, SeriesType): nodes = [] @@ -418,7 +429,7 @@ def _run_unary(self, assign, rhs): self._convert_series_calltype(rhs) out_data = ir.Var( arg.scope, mk_unique_var(assign.target.name + '_data'), rhs.loc) - self.typemap[out_data.name] = self.calltypes[rhs].return_type + self.state.typemap[out_data.name] = self.state.calltypes[rhs].return_type nodes.append(ir.Assign(rhs, out_data, rhs.loc)) return self._replace_func( lambda data: hpat.hiframes.api.init_series(data), @@ -429,11 +440,11 @@ def _run_unary(self, assign, rhs): return [assign] def _run_call(self, assign, lhs, rhs): - fdef = guard(find_callname, self.func_ir, rhs, self.typemap) + fdef = guard(find_callname, self.state.func_ir, rhs, self.state.typemap) if fdef is None: from numba.stencil import StencilFunc # could be make_function from list comprehension which is ok - func_def = guard(get_definition, self.func_ir, rhs.func) + func_def = guard(get_definition, self.state.func_ir, rhs.func) if isinstance(func_def, ir.Expr) and func_def.op == 'make_function': return [assign] if isinstance(func_def, ir.Global) and isinstance(func_def.value, StencilFunc): @@ -445,10 +456,10 @@ def _run_call(self, assign, lhs, rhs): func_name, func_mod = fdef if (isinstance(func_mod, ir.Var) - and self.typemap[func_mod.name] + and self.state.typemap[func_mod.name] == series_str_methods_type): - f_def = guard(get_definition, self.func_ir, rhs.func) - str_def = guard(get_definition, self.func_ir, f_def.value) + f_def = guard(get_definition, self.state.func_ir, rhs.func) + str_def = guard(get_definition, self.state.func_ir, f_def.value) if str_def is None: # TODO: check for errors raise ValueError("invalid series.str") @@ -460,7 +471,7 @@ def _run_call(self, assign, lhs, rhs): # arr.dtype transformation produces invalid code for dt64 # TODO: min if fdef == ('_get_type_max_value', 'hpat.hiframes.hiframes_typed'): - if self.typemap[rhs.args[0].name] == types.DType(types.NPDatetime('ns')): + if self.state.typemap[rhs.args[0].name] == types.DType(types.NPDatetime('ns')): return self._replace_func( lambda: hpat.hiframes.pd_timestamp_ext.integer_to_dt64( numba.targets.builtins.get_type_max_value( @@ -470,18 +481,18 @@ def _run_call(self, assign, lhs, rhs): d), rhs.args) if fdef == ('h5_read_dummy', 'hpat.io.pio_api'): - ndim = guard(find_const, self.func_ir, rhs.args[1]) - dtype_str = guard(find_const, self.func_ir, rhs.args[2]) + ndim = guard(find_const, self.state.func_ir, rhs.args[1]) + dtype_str = guard(find_const, self.state.func_ir, rhs.args[2]) index_var = rhs.args[3] filter_read = False func_text = "def _h5_read_impl(dset_id, ndim, dtype_str, index):\n" - if guard(is_whole_slice, self.typemap, self.func_ir, index_var): + if guard(is_whole_slice, self.state.typemap, self.state.func_ir, index_var): func_text += " size_0 = hpat.io.pio_api.h5size(dset_id, np.int32(0))\n" else: # TODO: check index format for this case filter_read = True - assert isinstance(self.typemap[index_var.name], types.BaseTuple) + assert isinstance(self.state.typemap[index_var.name], types.BaseTuple) func_text += " read_indices = hpat.io.pio_api.get_filter_read_indices(index[0])\n" func_text += " size_0 = len(read_indices)\n" for i in range(1, ndim): @@ -500,7 +511,7 @@ def _run_call(self, assign, lhs, rhs): func_text += " return A\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np}, loc_vars) _h5_read_impl = loc_vars['_h5_read_impl'] return self._replace_func(_h5_read_impl, rhs.args) @@ -508,14 +519,14 @@ def _run_call(self, assign, lhs, rhs): return self._run_pd_DatetimeIndex(assign, assign.target, rhs) if fdef == ('Series', 'pandas'): - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_series_ext.pd_series_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) if func_mod == 'hpat.hiframes.api': return self._run_call_hiframes(assign, assign.target, rhs, func_name) @@ -527,23 +538,23 @@ def _run_call(self, assign, lhs, rhs): return self._handle_empty_like(assign, lhs, rhs) if (isinstance(func_mod, ir.Var) - and is_series_type(self.typemap[func_mod.name])): + and is_series_type(self.state.typemap[func_mod.name])): return self._run_call_series( assign, assign.target, rhs, func_mod, func_name) if (isinstance(func_mod, ir.Var) and isinstance( - self.typemap[func_mod.name], SeriesRollingType)): + self.state.typemap[func_mod.name], SeriesRollingType)): return self._run_call_series_rolling( assign, assign.target, rhs, func_mod, func_name) if (isinstance(func_mod, ir.Var) and isinstance( - self.typemap[func_mod.name], DatetimeIndexType)): + self.state.typemap[func_mod.name], DatetimeIndexType)): return self._run_call_dt_index( assign, assign.target, rhs, func_mod, func_name) if (fdef == ('concat_dummy', 'hpat.hiframes.pd_dataframe_ext') - and isinstance(self.typemap[lhs], SeriesType)): + and isinstance(self.state.typemap[lhs], SeriesType)): return self._run_call_concat(assign, lhs, rhs) # handle sorted() with key lambda input @@ -558,19 +569,19 @@ def _run_call(self, assign, lhs, rhs): # example: test_sort_parallel if fdef == ('get_dataframe_data', 'hpat.hiframes.pd_dataframe_ext'): df_var = rhs.args[0] - df_typ = self.typemap[df_var.name] - ind = guard(find_const, self.func_ir, rhs.args[1]) - var_def = guard(get_definition, self.func_ir, df_var) - call_def = guard(find_callname, self.func_ir, var_def) + df_typ = self.state.typemap[df_var.name] + ind = guard(find_const, self.state.func_ir, rhs.args[1]) + var_def = guard(get_definition, self.state.func_ir, df_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_dataframe', 'hpat.hiframes.pd_dataframe_ext'): assign.value = var_def.args[ind] if fdef == ('get_dataframe_index', 'hpat.hiframes.pd_dataframe_ext'): df_var = rhs.args[0] - df_typ = self.typemap[df_var.name] + df_typ = self.state.typemap[df_var.name] n_cols = len(df_typ.columns) - var_def = guard(get_definition, self.func_ir, df_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, df_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_dataframe', 'hpat.hiframes.pd_dataframe_ext'): assign.value = var_def.args[n_cols] @@ -579,7 +590,7 @@ def _run_call(self, assign, lhs, rhs): nodes = [] new_args = [] for arg in rhs.args: - if isinstance(self.typemap[arg.name], SeriesType): + if isinstance(self.state.typemap[arg.name], SeriesType): new_args.append(self._get_series_data(arg, nodes)) else: new_args.append(arg) @@ -589,10 +600,10 @@ def _run_call(self, assign, lhs, rhs): # Second condition is to avoid chenging SeriesGroupBy class members # test: python -m hpat.runtests hpat.tests.test_series.TestSeries.test_series_groupby_count - if isinstance(self.typemap[lhs], SeriesType) and not isinstance(func_mod, ir.Var): + if isinstance(self.state.typemap[lhs], SeriesType) and not isinstance(func_mod, ir.Var): scope = assign.target.scope new_lhs = ir.Var(scope, mk_unique_var(lhs + '_data'), rhs.loc) - self.typemap[new_lhs.name] = self.calltypes[rhs].return_type + self.state.typemap[new_lhs.name] = self.state.calltypes[rhs].return_type nodes.append(ir.Assign(rhs, new_lhs, rhs.loc)) def _replace_func_param_impl(A): @@ -611,15 +622,15 @@ def _run_call_hiframes(self, assign, lhs, rhs, func_name): # remove the call since it is invalid for analysis here # XXX remove when df pass is typed? (test_pass_series2) if func_name == 'init_series' and isinstance( - self.typemap[rhs.args[0].name], SeriesType): + self.state.typemap[rhs.args[0].name], SeriesType): assign.value = rhs.args[0] return [assign] if func_name == 'get_index_data': # fix_df_array() calls get_index_data() for DatetimeIndex # but it can be removed sometimes - var_def = guard(get_definition, self.func_ir, rhs.args[0]) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, rhs.args[0]) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_datetime_index', 'hpat.hiframes.api'): assign.value = var_def.args[0] return [assign] @@ -627,8 +638,8 @@ def _run_call_hiframes(self, assign, lhs, rhs, func_name): if func_name == 'get_series_data': # fix_df_array() calls get_series_data() (e.g. for dataframes) # but it can be removed sometimes - var_def = guard(get_definition, self.func_ir, rhs.args[0]) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, rhs.args[0]) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_series', 'hpat.hiframes.api'): assign.value = var_def.args[0] return [assign] @@ -638,23 +649,23 @@ def _run_call_hiframes(self, assign, lhs, rhs, func_name): # arr = fix_df_array(col) -> arr=col if col is array if func_name == 'fix_df_array': - in_typ = self.typemap[rhs.args[0].name] + in_typ = self.state.typemap[rhs.args[0].name] impl = hpat.hiframes.api.fix_df_array_overload(in_typ) return self._replace_func(impl, rhs.args) # arr = fix_rolling_array(col) -> arr=col if col is float array if func_name == 'fix_rolling_array': in_arr = rhs.args[0] - if isinstance(self.typemap[in_arr.name].dtype, types.Float): + if isinstance(self.state.typemap[in_arr.name].dtype, types.Float): assign.value = rhs.args[0] return [assign] else: def f(column): # pragma: no cover a = column.astype(np.float64) f_block = compile_to_numba_ir(f, - {'hpat': hpat, 'np': np}, self.typingctx, - (if_series_to_array_type(self.typemap[in_arr.name]),), - self.typemap, self.calltypes).blocks.popitem()[1] + {'hpat': hpat, 'np': np}, self.state.typingctx, + (if_series_to_array_type(self.state.typemap[in_arr.name]),), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [in_arr]) nodes = f_block.body[:-3] nodes[-1].target = assign.target @@ -669,16 +680,16 @@ def f(column): # pragma: no cover new_tup = ir.Expr.build_tuple(tup_items, tup.loc) assign.value = new_tup # fix type and definition of lhs - self.typemap.pop(lhs.name) + self.state.typemap.pop(lhs.name) self._type_changed_vars.append(lhs.name) - self.typemap[lhs.name] = types.Tuple(tuple( - self.typemap[a.name] for a in tup_items)) + self.state.typemap[lhs.name] = types.Tuple(tuple( + self.state.typemap[a.name] for a in tup_items)) return [assign] if func_name == 'series_tup_to_arr_tup': - in_typ = self.typemap[rhs.args[0].name] + in_typ = self.state.typemap[rhs.args[0].name] assert isinstance(in_typ, types.BaseTuple), 'tuple expected' - series_vars = guard(get_definition, self.func_ir, rhs.args[0]).items + series_vars = guard(get_definition, self.state.func_ir, rhs.args[0]).items nodes = [] tup_items = [self._get_series_data(v, nodes) for v in series_vars] new_tup = ir.Expr.build_tuple(tup_items, lhs.loc) @@ -689,29 +700,29 @@ def f(column): # pragma: no cover if func_name == 'concat': # concat() case where tuple type changes by to_const_type() if any([a.name in self._type_changed_vars for a in rhs.args]): - argtyps = tuple(self.typemap[a.name] for a in rhs.args) - old_sig = self.calltypes.pop(rhs) - new_sig = self.typemap[rhs.func.name].get_call_type( - self.typingctx, argtyps, rhs.kws) - self.calltypes[rhs] = new_sig + argtyps = tuple(self.state.typemap[a.name] for a in rhs.args) + old_sig = self.state.calltypes.pop(rhs) + new_sig = self.state.typemap[rhs.func.name].get_call_type( + self.state.typingctx, argtyps, rhs.kws) + self.state.calltypes[rhs] = new_sig # replace tuple of Series with tuple of Arrays - in_vars, _ = guard(find_build_sequence, self.func_ir, rhs.args[0]) + in_vars, _ = guard(find_build_sequence, self.state.func_ir, rhs.args[0]) nodes = [] s_arrs = [self._get_series_data(v, nodes) if isinstance( - self.typemap[v.name], SeriesType) else v for v in in_vars] + self.state.typemap[v.name], SeriesType) else v for v in in_vars] new_tup = ir.Expr.build_tuple(s_arrs, lhs.loc) new_arg = ir.Var(lhs.scope, mk_unique_var( rhs.args[0].name + '_arrs'), lhs.loc) - self.typemap[new_arg.name] = if_series_to_array_type( - self.typemap[rhs.args[0].name]) + self.state.typemap[new_arg.name] = if_series_to_array_type( + self.state.typemap[rhs.args[0].name]) nodes.append(ir.Assign(new_tup, new_arg, lhs.loc)) rhs.args[0] = new_arg nodes.append(assign) - self.calltypes.pop(rhs) - new_sig = self.typemap[rhs.func.name].get_call_type( - self.typingctx, (self.typemap[new_arg.name],), rhs.kws) - self.calltypes[rhs] = new_sig + self.state.calltypes.pop(rhs) + new_sig = self.state.typemap[rhs.func.name].get_call_type( + self.state.typingctx, (self.state.typemap[new_arg.name],), rhs.kws) + self.state.calltypes[rhs] = new_sig return nodes # replace isna early to enable more optimization in PA @@ -719,7 +730,7 @@ def f(column): # pragma: no cover if func_name == 'isna': arr = rhs.args[0] ind = rhs.args[1] - arr_typ = self.typemap[arr.name] + arr_typ = self.state.typemap[arr.name] if isinstance(arr_typ, (types.Array, SeriesType)): if isinstance(arr_typ.dtype, types.Float): def func(arr, i): @@ -770,7 +781,7 @@ def _isin_series(A, vals): if func_name == 'flatten_to_series': arg = rhs.args[0] - in_typ = self.typemap[arg.name] + in_typ = self.state.typemap[arg.name] nodes = [] if isinstance(in_typ, SeriesType): arg = self._get_series_data(arg, nodes) @@ -789,7 +800,7 @@ def _flatten_impl(A): return self._replace_func(_flatten_impl, [arg], pre_nodes=nodes) if func_name == 'to_numeric': - out_dtype = self.typemap[lhs.name].dtype + out_dtype = self.state.typemap[lhs.name].dtype assert out_dtype == types.int64 or out_dtype == types.float64 # TODO: handle non-Series input @@ -828,7 +839,7 @@ def parse_impl(data): nodes = [] new_args = [] for arg in rhs.args: - if isinstance(self.typemap[arg.name], SeriesType): + if isinstance(self.state.typemap[arg.name], SeriesType): new_args.append(self._get_series_data(arg, nodes)) else: new_args.append(arg) @@ -848,7 +859,7 @@ def _run_call_series(self, assign, lhs, rhs, series_var, func_name): raise ValueError("HPAT pipeline does not support arguments for Series.{}()".format(func_name)) # TODO: handle skipna, min_count arguments - series_typ = self.typemap[series_var.name] + series_typ = self.state.typemap[series_var.name] series_dtype = series_typ.dtype func = series_replace_funcs[func_name] if isinstance(func, dict): @@ -946,14 +957,14 @@ def run_call_series_quantile_default(A): if n_arg is False: n_arg = ir.Var(lhs.scope, mk_unique_var('head_n'), lhs.loc) # default is 5 - self.typemap[n_arg.name] = types.IntegerLiteral(5) + self.state.typemap[n_arg.name] = types.IntegerLiteral(5) nodes.append(ir.Assign( ir.Const(5, lhs.loc), n_arg, lhs.loc)) data = self._get_series_data(series_var, nodes) func = series_replace_funcs[func_name] - if self.typemap[series_var.name].index != types.none: + if self.state.typemap[series_var.name].index != types.none: index = self._get_series_index(series_var, nodes) func = series_replace_funcs['head_index'] else: @@ -975,7 +986,7 @@ def run_call_series_quantile_default(A): if func_name == 'rolling': # XXX: remove rolling setup call, assuming still available in definitions - self.func_ir._definitions[lhs.name].append(rhs) + self.state.func_ir._definitions[lhs.name].append(rhs) return [] if func_name == 'combine': @@ -988,7 +999,7 @@ def run_call_series_quantile_default(A): nodes = [] data = self._get_series_data(series_var, nodes) other = rhs.args[0] - if isinstance(self.typemap[other.name], SeriesType): + if isinstance(self.state.typemap[other.name], SeriesType): func = series_replace_funcs['append_single'] other = self._get_series_data(other, nodes) else: @@ -1008,9 +1019,9 @@ def run_call_series_quantile_default(A): # data of input becomes both key and data for aggregate input # data of output is the counts out_key_var = ir.Var(lhs.scope, mk_unique_var(lhs.name + '_index'), lhs.loc) - self.typemap[out_key_var.name] = self.typemap[data.name] + self.state.typemap[out_key_var.name] = self.state.typemap[data.name] out_data_var = ir.Var(lhs.scope, mk_unique_var(lhs.name + '_data'), lhs.loc) - self.typemap[out_data_var.name] = self.typemap[lhs.name].data + self.state.typemap[out_data_var.name] = self.state.typemap[lhs.name].data agg_func = series_replace_funcs['count'] agg_node = hiframes.aggregate.Aggregate( lhs.name, 'series', ['series'], [out_key_var], { @@ -1024,9 +1035,9 @@ def func(A, B): return self._replace_func(func, [out_data_var, out_key_var], pre_nodes=nodes) # astype with string output - if func_name == 'astype' and is_str_series_typ(self.typemap[lhs.name]): + if func_name == 'astype' and is_str_series_typ(self.state.typemap[lhs.name]): # just return input if string - if is_str_series_typ(self.typemap[series_var.name]): + if is_str_series_typ(self.state.typemap[series_var.name]): return self._replace_func(lambda a: a, [series_var]) func = series_replace_funcs['astype_str'] nodes = [] @@ -1063,7 +1074,7 @@ def func(A, B): ).format(func_name, arg_names) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _func_impl = loc_vars['_func_impl'] return self._replace_func(_func_impl, [data] + rhs.args, pre_nodes=nodes) @@ -1078,16 +1089,16 @@ def _handle_series_sort(self, lhs, rhs, series_var, is_argsort): data = self._get_series_data(series_var, nodes) # get index array - if self.typemap[series_var.name].index != types.none: + if self.state.typemap[series_var.name].index != types.none: index_var = self._get_series_index(series_var, nodes) else: index_var = self._get_index_values(data, nodes) # output data arrays for results, before conversion to Series out_data = ir.Var(lhs.scope, mk_unique_var(lhs.name + '_data'), lhs.loc) - self.typemap[out_data.name] = self.typemap[lhs.name].data + self.state.typemap[out_data.name] = self.state.typemap[lhs.name].data out_index = ir.Var(lhs.scope, mk_unique_var(lhs.name + '_index'), lhs.loc) - self.typemap[out_index.name] = self.typemap[index_var.name] + self.state.typemap[out_index.name] = self.state.typemap[index_var.name] # indexes are input/output Sort data in_df = {'inds': index_var} @@ -1100,7 +1111,7 @@ def _handle_series_sort(self, lhs, rhs, series_var, is_argsort): if is_argsort: # output of argsort doesn't have new index so assign None none_index = ir.Var(lhs.scope, mk_unique_var(lhs.name + '_index'), lhs.loc) - self.typemap[none_index.name] = types.none + self.state.typemap[none_index.name] = types.none nodes.append(ir.Assign( ir.Const(None, lhs.loc), none_index, lhs.loc)) args = [out_index, none_index] @@ -1111,7 +1122,7 @@ def _handle_series_sort(self, lhs, rhs, series_var, is_argsort): 'sort_values', rhs.args, dict(rhs.kws), 1, 'ascending', default=True) if isinstance(ascending, ir.Var): # TODO: check error - ascending = find_const(self.func_ir, ascending) + ascending = find_const(self.state.func_ir, ascending) # Sort node nodes.append(hiframes.sort.Sort(data.name, lhs.name, in_keys, @@ -1124,7 +1135,7 @@ def _handle_series_sort(self, lhs, rhs, series_var, is_argsort): pre_nodes=nodes) def _run_call_series_fillna(self, assign, lhs, rhs, series_var): - dtype = self.typemap[series_var.name].dtype + dtype = self.state.typemap[series_var.name].dtype val = rhs.args[0] nodes = [] data = self._get_series_data(series_var, nodes) @@ -1132,14 +1143,14 @@ def _run_call_series_fillna(self, assign, lhs, rhs, series_var): kws = dict(rhs.kws) inplace = False if 'inplace' in kws: - inplace = guard(find_const, self.func_ir, kws['inplace']) + inplace = guard(find_const, self.state.func_ir, kws['inplace']) if inplace is None: # pragma: no cover raise ValueError("inplace arg to fillna should be constant") if inplace: if dtype == string_type: # optimization: just set null bit if fill is empty - if guard(find_const, self.func_ir, val) == "": + if guard(find_const, self.state.func_ir, val) == "": return self._replace_func( lambda A: hpat.str_arr_ext.set_null_bits(A), [data], @@ -1170,11 +1181,11 @@ def str_fillna_impl(A, fill, name): return self._replace_func(func, [data, val, name], pre_nodes=nodes) def _run_call_series_dropna(self, assign, lhs, rhs, series_var): - dtype = self.typemap[series_var.name].dtype + dtype = self.state.typemap[series_var.name].dtype kws = dict(rhs.kws) inplace = False if 'inplace' in kws: - inplace = guard(find_const, self.func_ir, kws['inplace']) + inplace = guard(find_const, self.state.func_ir, kws['inplace']) if inplace is None: # pragma: no cover raise ValueError("inplace arg to dropna should be constant") @@ -1210,15 +1221,15 @@ def _handle_series_map(self, assign, lhs, rhs, series_var): # error checking: make sure there is function input only if len(rhs.args) != 1: raise ValueError("map expects 1 argument") - func = guard(get_definition, self.func_ir, rhs.args[0]) + func = guard(get_definition, self.state.func_ir, rhs.args[0]) if func is None or not (isinstance(func, ir.Expr) and func.op == 'make_function'): raise ValueError("lambda for map not found") - dtype = self.typemap[series_var.name].dtype + dtype = self.state.typemap[series_var.name].dtype nodes = [] data = self._get_series_data(series_var, nodes) - out_typ = self.typemap[lhs.name].dtype + out_typ = self.state.typemap[lhs.name].dtype # TODO: handle non numpy alloc types like string array # prange func to inline @@ -1243,10 +1254,10 @@ def _handle_series_map(self, assign, lhs, rhs, series_var): #func_text += " return ret\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] - _globals = self.func_ir.func_id.func.__globals__ + _globals = self.state.func_ir.func_id.func.__globals__ f_ir = compile_to_numba_ir(f, {'numba': numba, 'np': np, 'hpat': hpat}) # fix definitions to enable finding sentinel @@ -1262,21 +1273,22 @@ def _handle_series_map(self, assign, lhs, rhs, series_var): and stmt.value.op == 'call'): fdef = guard(get_definition, f_ir, stmt.value.func) if isinstance(fdef, ir.Global) and fdef.name == 'map_func': + update_globals(func, _globals) inline_closure_call(f_ir, _globals, block, i, func) break # remove sentinel global to avoid type inference issues ir_utils.remove_dead(f_ir.blocks, f_ir.arg_names, f_ir) f_ir._definitions = build_definitions(f_ir.blocks) - arg_typs = (self.typemap[data.name],) - f_typemap, _f_ret_t, f_calltypes = numba.compiler.type_inference_stage( - self.typingctx, f_ir, arg_typs, None) + arg_typs = (self.state.typemap[data.name],) + f_typemap, _f_ret_t, f_calltypes = numba.typed_passes.type_inference_stage( + self.state.typingctx, f_ir, arg_typs, None) # remove argument entries like arg.a from typemap arg_names = [vname for vname in f_typemap if vname.startswith("arg.")] for a in arg_names: f_typemap.pop(a) - self.typemap.update(f_typemap) - self.calltypes.update(f_calltypes) + self.state.typemap.update(f_typemap) + self.state.calltypes.update(f_calltypes) replace_arg_nodes(f_ir.blocks[topo_order[0]], [data]) f_ir.blocks[topo_order[0]].body = nodes + f_ir.blocks[topo_order[0]].body return f_ir.blocks @@ -1286,7 +1298,7 @@ def _run_call_rolling(self, assign, lhs, rhs, func_name): nodes = [] new_args = [] for arg in rhs.args: - if isinstance(self.typemap[arg.name], SeriesType): + if isinstance(self.state.typemap[arg.name], SeriesType): new_args.append(self._get_series_data(arg, nodes)) else: new_args.append(arg) @@ -1326,12 +1338,12 @@ def rolling_cov_impl(arr, other, w, center): # pragma: no cover rolling_cov_impl, rhs.args, pre_nodes=nodes) # replace apply function with dispatcher obj, now the type is known if (func_name == 'rolling_fixed' and isinstance( - self.typemap[rhs.args[4].name], types.MakeFunctionLiteral)): + self.state.typemap[rhs.args[4].name], types.MakeFunctionLiteral)): # for apply case, create a dispatcher for the kernel and pass it # TODO: automatically handle lambdas in Numba - dtype = self.typemap[rhs.args[0].name].dtype - out_dtype = self.typemap[lhs.name].dtype - func_node = guard(get_definition, self.func_ir, rhs.args[4]) + dtype = self.state.typemap[rhs.args[0].name].dtype + out_dtype = self.state.typemap[lhs.name].dtype + func_node = guard(get_definition, self.state.func_ir, rhs.args[4]) imp_dis = self._handle_rolling_apply_func( func_node, dtype, out_dtype) @@ -1339,20 +1351,20 @@ def f(arr, w, center): # pragma: no cover df_arr = hpat.hiframes.rolling.rolling_fixed( arr, w, center, False, _func) f_block = compile_to_numba_ir(f, {'hpat': hpat, '_func': imp_dis}, - self.typingctx, - tuple(self.typemap[v.name] for v in rhs.args[:-2]), - self.typemap, self.calltypes).blocks.popitem()[1] + self.state.typingctx, + tuple(self.state.typemap[v.name] for v in rhs.args[:-2]), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, rhs.args[:-2]) nodes += f_block.body[:-3] # remove none return nodes[-1].target = lhs return nodes elif (func_name == 'rolling_variable' and isinstance( - self.typemap[rhs.args[5].name], types.MakeFunctionLiteral)): + self.state.typemap[rhs.args[5].name], types.MakeFunctionLiteral)): # for apply case, create a dispatcher for the kernel and pass it # TODO: automatically handle lambdas in Numba - dtype = self.typemap[rhs.args[0].name].dtype - out_dtype = self.typemap[lhs.name].dtype - func_node = guard(get_definition, self.func_ir, rhs.args[5]) + dtype = self.state.typemap[rhs.args[0].name].dtype + out_dtype = self.state.typemap[lhs.name].dtype + func_node = guard(get_definition, self.state.func_ir, rhs.args[5]) imp_dis = self._handle_rolling_apply_func( func_node, dtype, out_dtype) @@ -1360,9 +1372,9 @@ def f(arr, on_arr, w, center): # pragma: no cover df_arr = hpat.hiframes.rolling.rolling_variable( arr, on_arr, w, center, False, _func) f_block = compile_to_numba_ir(f, {'hpat': hpat, '_func': imp_dis}, - self.typingctx, - tuple(self.typemap[v.name] for v in rhs.args[:-2]), - self.typemap, self.calltypes).blocks.popitem()[1] + self.state.typingctx, + tuple(self.state.typemap[v.name] for v in rhs.args[:-2]), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, rhs.args[:-2]) nodes += f_block.body[:-3] # remove none return nodes[-1].target = lhs @@ -1379,12 +1391,12 @@ def _handle_series_combine(self, assign, lhs, rhs, series_var): raise ValueError("not enough arguments in call to combine") if len(rhs.args) > 3: raise ValueError("too many arguments in call to combine") - func = guard(get_definition, self.func_ir, rhs.args[1]) + func = guard(get_definition, self.state.func_ir, rhs.args[1]) if func is None or not (isinstance(func, ir.Expr) and func.op == 'make_function'): raise ValueError("lambda for combine not found") - out_typ = self.typemap[lhs.name].dtype + out_typ = self.state.typemap[lhs.name].dtype other = rhs.args[0] nodes = [] data = self._get_series_data(series_var, nodes) @@ -1402,14 +1414,14 @@ def _handle_series_combine(self, assign, lhs, rhs, series_var): func_text += " n1 = len(A)\n" func_text += " n2 = len(B)\n" func_text += " n = max(n1, n2)\n" - if not isinstance(self.typemap[series_var.name].dtype, types.Float) and use_nan: + if not isinstance(self.state.typemap[series_var.name].dtype, types.Float) and use_nan: func_text += " assert n1 == n, 'can not use NAN for non-float series, with different length'\n" - if not isinstance(self.typemap[other.name].dtype, types.Float) and use_nan: + if not isinstance(self.state.typemap[other.name].dtype, types.Float) and use_nan: func_text += " assert n2 == n, 'can not use NAN for non-float series, with different length'\n" func_text += " numba.parfor.init_prange()\n" func_text += " S = numba.unsafe.ndarray.empty_inferred((n,))\n" func_text += " for i in numba.parfor.internal_prange(n):\n" - if use_nan and isinstance(self.typemap[series_var.name].dtype, types.Float): + if use_nan and isinstance(self.state.typemap[series_var.name].dtype, types.Float): func_text += " t1 = np.nan\n" func_text += " if i < n1:\n" func_text += " t1 = A[i]\n" @@ -1421,7 +1433,7 @@ def _handle_series_combine(self, assign, lhs, rhs, series_var): func_text += " if i < n1:\n" func_text += " t1 = A[i]\n" # same, but for 2nd argument - if use_nan and isinstance(self.typemap[other.name].dtype, types.Float): + if use_nan and isinstance(self.state.typemap[other.name].dtype, types.Float): func_text += " t2 = np.nan\n" func_text += " if i < n2:\n" func_text += " t2 = B[i]\n" @@ -1435,10 +1447,10 @@ def _handle_series_combine(self, assign, lhs, rhs, series_var): func_text += " return hpat.hiframes.api.init_series(S)\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] - _globals = self.func_ir.func_id.func.__globals__ + _globals = self.state.func_ir.func_id.func.__globals__ f_ir = compile_to_numba_ir(f, {'numba': numba, 'np': np, 'hpat': hpat}) # fix definitions to enable finding sentinel @@ -1454,23 +1466,24 @@ def _handle_series_combine(self, assign, lhs, rhs, series_var): and stmt.value.op == 'call'): fdef = guard(get_definition, f_ir, stmt.value.func) if isinstance(fdef, ir.Global) and fdef.name == 'map_func': + update_globals(func, _globals) inline_closure_call(f_ir, _globals, block, i, func) break # remove sentinel global to avoid type inference issues ir_utils.remove_dead(f_ir.blocks, f_ir.arg_names, f_ir) f_ir._definitions = build_definitions(f_ir.blocks) - arg_typs = (self.typemap[data.name], self.typemap[other_data.name],) + arg_typs = (self.state.typemap[data.name], self.state.typemap[other_data.name],) if not use_nan: - arg_typs += (self.typemap[rhs.args[2].name],) - f_typemap, _f_ret_t, f_calltypes = numba.compiler.type_inference_stage( - self.typingctx, f_ir, arg_typs, None) + arg_typs += (self.state.typemap[rhs.args[2].name],) + f_typemap, _f_ret_t, f_calltypes = numba.typed_passes.type_inference_stage( + self.state.typingctx, f_ir, arg_typs, None) # remove argument entries like arg.a from typemap arg_names = [vname for vname in f_typemap if vname.startswith("arg.")] for a in arg_names: f_typemap.pop(a) - self.typemap.update(f_typemap) - self.calltypes.update(f_calltypes) + self.state.typemap.update(f_typemap) + self.state.calltypes.update(f_calltypes) func_args = [data, other_data] if not use_nan: func_args.append(rhs.args[2]) @@ -1484,18 +1497,18 @@ def _run_call_series_rolling(self, assign, lhs, rhs, rolling_var, func_name): Handle Series rolling calls like: A = df.column.rolling(3).sum() """ - rolling_call = guard(get_definition, self.func_ir, rolling_var) + rolling_call = guard(get_definition, self.state.func_ir, rolling_var) assert isinstance(rolling_call, ir.Expr) and rolling_call.op == 'call' - call_def = guard(get_definition, self.func_ir, rolling_call.func) + call_def = guard(get_definition, self.state.func_ir, rolling_call.func) assert isinstance(call_def, ir.Expr) and call_def.op == 'getattr' series_var = call_def.value nodes = [] data = self._get_series_data(series_var, nodes) - window, center, on = get_rolling_setup_args(self.func_ir, rolling_call, False) + window, center, on = get_rolling_setup_args(self.state.func_ir, rolling_call, False) if not isinstance(center, ir.Var): center_var = ir.Var(lhs.scope, mk_unique_var("center"), lhs.loc) - self.typemap[center_var.name] = types.bool_ + self.state.typemap[center_var.name] = types.bool_ nodes.append(ir.Assign(ir.Const(center, lhs.loc), center_var, lhs.loc)) center = center_var @@ -1514,9 +1527,9 @@ def f(a, b, w, c): return self._replace_func(f, [data, other, window, center], pre_nodes=nodes) elif func_name == 'apply': - func_node = guard(get_definition, self.func_ir, rhs.args[0]) - dtype = self.typemap[data.name].dtype - out_dtype = self.typemap[lhs.name].dtype + func_node = guard(get_definition, self.state.func_ir, rhs.args[0]) + dtype = self.state.typemap[data.name].dtype + out_dtype = self.state.typemap[lhs.name].dtype func_global = self._handle_rolling_apply_func(func_node, dtype, out_dtype) else: func_global = func_name @@ -1536,7 +1549,7 @@ def _handle_rolling_apply_func(self, func_node, dtype, out_dtype): if func_node.defaults is not None: raise ValueError("rolling apply kernel functions cannot have default arguments") # create a function from the code object - glbs = self.func_ir.func_id.func.__globals__ + glbs = self.state.func_ir.func_id.func.__globals__ lcs = {} exec("def f(A): return A", glbs, lcs) kernel_func = lcs['f'] @@ -1557,7 +1570,7 @@ def _run_DatetimeIndex_field(self, assign, lhs, rhs): """transform DatetimeIndex. and Series.dt. """ nodes = [] - in_typ = self.typemap[rhs.value.name] + in_typ = self.state.typemap[rhs.value.name] if isinstance(in_typ, DatetimeIndexType): arr = self._get_dt_index_data(rhs.value, nodes) is_dt_index = True @@ -1581,7 +1594,7 @@ def _run_DatetimeIndex_field(self, assign, lhs, rhs): else: func_text += ' return hpat.hiframes.api.init_series(S)\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arr], pre_nodes=nodes) @@ -1590,7 +1603,7 @@ def _run_DatetimeIndex_date(self, assign, lhs, rhs): """transform DatetimeIndex.date and Series.dt.date """ nodes = [] - in_typ = self.typemap[rhs.value.name] + in_typ = self.state.typemap[rhs.value.name] if isinstance(in_typ, DatetimeIndexType): arr = self._get_dt_index_data(rhs.value, nodes) is_dt_index = True @@ -1613,7 +1626,7 @@ def _run_DatetimeIndex_date(self, assign, lhs, rhs): else: func_text += ' return hpat.hiframes.api.init_series(S)\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arr], pre_nodes=nodes) @@ -1654,7 +1667,7 @@ def _run_Timedelta_field(self, assign, lhs, rhs): assert(0) func_text += ' return S\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arr], pre_nodes=nodes) @@ -1662,13 +1675,13 @@ def _run_Timedelta_field(self, assign, lhs, rhs): def _run_pd_DatetimeIndex(self, assign, lhs, rhs): """transform pd.DatetimeIndex() call with string array argument """ - arg_typs = tuple(self.typemap[v.name] for v in rhs.args) - kw_typs = {name: self.typemap[v.name] + arg_typs = tuple(self.state.typemap[v.name] for v in rhs.args) + kw_typs = {name: self.state.typemap[v.name] for name, v in dict(rhs.kws).items()} impl = hpat.hiframes.pd_index_ext.pd_datetimeindex_overload( *arg_typs, **kw_typs) return self._replace_func(impl, rhs.args, - pysig=self.calltypes[rhs].pysig, kws=dict(rhs.kws)) + pysig=self.state.calltypes[rhs].pysig, kws=dict(rhs.kws)) def _run_series_str_method(self, assign, lhs, series_var, func_name, rhs): @@ -1699,7 +1712,7 @@ def _run_series_str_method(self, assign, lhs, series_var, func_name, rhs): func_text += ' return hpat.hiframes.api.init_series(S)\n' loc_vars = {} # print(func_text) - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arr], pre_nodes=nodes, extra_globals={ @@ -1733,7 +1746,7 @@ def _run_series_str_method(self, assign, lhs, series_var, func_name, rhs): func_text += ' S[i] = len(val)\n' func_text += ' return hpat.hiframes.api.init_series(S)\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arr], pre_nodes=nodes) @@ -1743,7 +1756,7 @@ def _run_series_str_replace(self, assign, lhs, arr, rhs, nodes): # TODO: refactor arg parsing kws = dict(rhs.kws) if 'regex' in kws: - regex = guard(find_const, self.func_ir, kws['regex']) + regex = guard(find_const, self.state.func_ir, kws['regex']) if regex is None: raise ValueError( "str.replace regex argument should be constant") @@ -1767,11 +1780,11 @@ def _run_series_str_split(self, assign, lhs, arr, rhs, nodes): if sep is False: sep = ir.Var(lhs.scope, mk_unique_var('split_sep'), lhs.loc) sep_typ = types.none - self.typemap[sep.name] = types.none + self.state.typemap[sep.name] = types.none nodes.append(ir.Assign( ir.Const(None, lhs.loc), sep, lhs.loc)) else: - sep_typ = self.typemap[sep.name] + sep_typ = self.state.typemap[sep.name] def _str_split_impl(str_arr, sep): numba.parfor.init_prange() @@ -1792,7 +1805,7 @@ def _str_split_impl(str_arr, sep): return self._replace_func(_str_split_impl, [arr, sep], pre_nodes=nodes) def _run_series_str_get(self, assign, lhs, arr, rhs, nodes): - arr_typ = self.typemap[arr.name] + arr_typ = self.state.typemap[arr.name] # XXX only supports get for list(list(str)) input and split view assert (arr_typ == types.List(types.List(string_type)) or arr_typ == string_array_split_view_type) @@ -1849,7 +1862,7 @@ def _is_dt_index_binop(self, rhs): if rhs.fn not in _dt_index_binops: return False - arg1, arg2 = self.typemap[rhs.lhs.name], self.typemap[rhs.rhs.name] + arg1, arg2 = self.state.typemap[rhs.lhs.name], self.state.typemap[rhs.rhs.name] # one of them is dt_index but not both if ((is_dt64_series_typ(arg1) or is_dt64_series_typ(arg2)) and not (is_dt64_series_typ(arg1) and is_dt64_series_typ(arg2))): @@ -1868,15 +1881,15 @@ def _is_allowed_type(t): return is_dt64_series_typ(t) or t == string_type # TODO: this has to be more generic to support all combinations. - if (is_dt64_series_typ(self.typemap[arg1.name]) - and self.typemap[arg2.name] == hpat.hiframes.pd_timestamp_ext.pandas_timestamp_type + if (is_dt64_series_typ(self.state.typemap[arg1.name]) + and self.state.typemap[arg2.name] == hpat.hiframes.pd_timestamp_ext.pandas_timestamp_type and rhs.fn in ('-', operator.sub)): return self._replace_func( series_kernels._column_sub_impl_datetime_series_timestamp, [arg1, arg2]) - if (isinstance(self.typemap[arg1.name], DatetimeIndexType) - and self.typemap[arg2.name] == hpat.hiframes.pd_timestamp_ext.pandas_timestamp_type + if (isinstance(self.state.typemap[arg1.name], DatetimeIndexType) + and self.state.typemap[arg2.name] == hpat.hiframes.pd_timestamp_ext.pandas_timestamp_type and rhs.fn in ('-', operator.sub)): nodes = [] arg1 = self._get_dt_index_data(arg1, nodes) @@ -1884,14 +1897,14 @@ def _is_allowed_type(t): series_kernels._column_sub_impl_datetimeindex_timestamp, [ arg1, arg2], pre_nodes=nodes) - if (not _is_allowed_type(types.unliteral(self.typemap[arg1.name])) - or not _is_allowed_type(types.unliteral(self.typemap[arg2.name]))): + if (not _is_allowed_type(types.unliteral(self.state.typemap[arg1.name])) + or not _is_allowed_type(types.unliteral(self.state.typemap[arg2.name]))): raise ValueError("DatetimeIndex operation not supported") # string comparison with DatetimeIndex op_str = _binop_to_str[rhs.fn] - typ1 = self.typemap[arg1.name] - typ2 = self.typemap[arg2.name] + typ1 = self.state.typemap[arg1.name] + typ2 = self.state.typemap[arg2.name] nodes = [] is_out_series = False @@ -1922,7 +1935,7 @@ def _is_allowed_type(t): else: func_text += ' return S\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'numba': numba}, loc_vars) f = loc_vars['f'] # print(func_text) return self._replace_func(f, [arg1, arg2]) @@ -1930,33 +1943,33 @@ def _is_allowed_type(t): def _handle_string_array_expr(self, assign, rhs): # convert str_arr==str into parfor if (rhs.fn in _string_array_comp_ops - and is_str_arr_typ(self.typemap[rhs.lhs.name]) - or is_str_arr_typ(self.typemap[rhs.rhs.name])): + and is_str_arr_typ(self.state.typemap[rhs.lhs.name]) + or is_str_arr_typ(self.state.typemap[rhs.rhs.name])): nodes = [] arg1 = rhs.lhs arg2 = rhs.rhs is_series = False - if is_str_series_typ(self.typemap[arg1.name]): + if is_str_series_typ(self.state.typemap[arg1.name]): arg1 = self._get_series_data(arg1, nodes) is_series = True - if is_str_series_typ(self.typemap[arg2.name]): + if is_str_series_typ(self.state.typemap[arg2.name]): arg2 = self._get_series_data(arg2, nodes) is_series = True arg1_access = 'A' arg2_access = 'B' len_call = 'len(A)' - if is_str_arr_typ(self.typemap[arg1.name]): + if is_str_arr_typ(self.state.typemap[arg1.name]): arg1_access = 'A[i]' # replace type now for correct typing of len, etc. - self.typemap.pop(arg1.name) - self.typemap[arg1.name] = string_array_type + self.state.typemap.pop(arg1.name) + self.state.typemap[arg1.name] = string_array_type - if is_str_arr_typ(self.typemap[arg2.name]): + if is_str_arr_typ(self.state.typemap[arg2.name]): arg1_access = 'B[i]' len_call = 'len(B)' - self.typemap.pop(arg2.name) - self.typemap[arg2.name] = string_array_type + self.state.typemap.pop(arg2.name) + self.state.typemap[arg2.name] = string_array_type op_str = _binop_to_str[rhs.fn] @@ -1972,7 +1985,7 @@ def _handle_string_array_expr(self, assign, rhs): func_text += ' return S\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, [arg1, arg2], pre_nodes=nodes) @@ -1987,7 +2000,7 @@ def _run_series_str_contains(self, rhs, series_var, nodes): pat = rhs.args[0] regex = True # default regex arg is True if 'regex' in kws: - regex = guard(find_const, self.func_ir, kws['regex']) + regex = guard(find_const, self.state.func_ir, kws['regex']) if regex is None: raise ValueError("str.contains expects constant regex argument") if regex: @@ -2002,7 +2015,7 @@ def _handle_empty_like(self, assign, lhs, rhs): # B = empty_like(A) -> B = empty(len(A), dtype) in_arr = rhs.args[0] - if self.typemap[in_arr.name].ndim == 1: + if self.state.typemap[in_arr.name].ndim == 1: # generate simpler len() for 1D case def f(_in_arr): # pragma: no cover _alloc_size = len(_in_arr) @@ -2012,8 +2025,8 @@ def f(_in_arr): # pragma: no cover _alloc_size = _in_arr.shape _out_arr = np.empty(_alloc_size, _in_arr.dtype) - f_block = compile_to_numba_ir(f, {'np': np}, self.typingctx, (if_series_to_array_type( - self.typemap[in_arr.name]),), self.typemap, self.calltypes).blocks.popitem()[1] + f_block = compile_to_numba_ir(f, {'np': np}, self.state.typingctx, (if_series_to_array_type( + self.state.typemap[in_arr.name]),), self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [in_arr]) nodes = f_block.body[:-3] # remove none return nodes[-1].target = assign.target @@ -2035,7 +2048,7 @@ def _handle_str_contains(self, assign, lhs, rhs, fname): func_text += ' S[i] = {}(str_arr[i], pat)\n'.format(comp_func) func_text += ' return hpat.hiframes.api.init_series(S)\n' loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) f = loc_vars['f'] return self._replace_func(f, rhs.args) @@ -2043,9 +2056,9 @@ def _handle_df_col_filter(self, assign, lhs, rhs): nodes = [] in_arr = rhs.args[0] bool_arr = rhs.args[1] - if is_series_type(self.typemap[in_arr.name]): + if is_series_type(self.state.typemap[in_arr.name]): in_arr = self._get_series_data(in_arr, nodes) - if is_series_type(self.typemap[bool_arr.name]): + if is_series_type(self.state.typemap[bool_arr.name]): bool_arr = self._get_series_data(bool_arr, nodes) return self._replace_func(series_kernels._column_filter_impl, @@ -2068,9 +2081,9 @@ def _handle_df_col_calls(self, assign, lhs, rhs, func_name): if func_name == 'dropna': # df.dropna case - if isinstance(self.typemap[rhs.args[0].name], types.BaseTuple): + if isinstance(self.state.typemap[rhs.args[0].name], types.BaseTuple): return self._handle_df_dropna(assign, lhs, rhs) - dtype = self.typemap[rhs.args[0].name].dtype + dtype = self.state.typemap[rhs.args[0].name].dtype if dtype == string_type: func = series_replace_funcs['dropna_str_alloc'] elif isinstance(dtype, types.Float): @@ -2093,9 +2106,9 @@ def func(A): return [assign] def _handle_df_dropna(self, assign, lhs, rhs): - in_typ = self.typemap[rhs.args[0].name] + in_typ = self.state.typemap[rhs.args[0].name] - in_vars, _ = guard(find_build_sequence, self.func_ir, rhs.args[0]) + in_vars, _ = guard(find_build_sequence, self.state.func_ir, rhs.args[0]) in_names = [mk_unique_var(in_vars[i].name).replace('.', '_') for i in range(len(in_vars))] out_names = [mk_unique_var(in_vars[i].name).replace('.', '_') for i in range(len(in_vars))] str_colnames = [in_names[i] for i, t in enumerate(in_typ.types) if is_str_arr_typ(t)] @@ -2135,16 +2148,16 @@ def _handle_df_dropna(self, assign, lhs, rhs): func_text += " return ({},)\n".format(", ".join(out_names)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np, 'numba': numba}, loc_vars) _dropna_impl = loc_vars['_dropna_impl'] return self._replace_func(_dropna_impl, rhs.args) def _run_call_concat(self, assign, lhs, rhs): nodes = [] - series_list = guard(get_definition, self.func_ir, rhs.args[0]).items + series_list = guard(get_definition, self.state.func_ir, rhs.args[0]).items arrs = [self._get_series_data(v, nodes) for v in series_list] arr_tup = ir.Var(rhs.args[0].scope, mk_unique_var('arr_tup'), rhs.args[0].loc) - self.typemap[arr_tup.name] = types.Tuple([self.typemap[a.name] for a in arrs]) + self.state.typemap[arr_tup.name] = types.Tuple([self.state.typemap[a.name] for a in arrs]) tup_expr = ir.Expr.build_tuple(arrs, arr_tup.loc) nodes.append(ir.Assign(tup_expr, arr_tup, arr_tup.loc)) return self._replace_func( @@ -2154,8 +2167,8 @@ def _run_call_concat(self, assign, lhs, rhs): def _handle_h5_write(self, dset, index, arr): if index != slice(None): raise ValueError("Only HDF5 write of full array supported") - assert isinstance(self.typemap[arr.name], types.Array) - ndim = self.typemap[arr.name].ndim + assert isinstance(self.state.typemap[arr.name], types.Array) + ndim = self.state.typemap[arr.name].ndim func_text = "def _h5_write_impl(dset_id, arr):\n" func_text += " zero_tup = ({},)\n".format(", ".join(["0"] * ndim)) @@ -2165,16 +2178,16 @@ def _handle_h5_write(self, dset, index, arr): func_text += " zero_tup, arr_shape, 0, arr)\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np}, loc_vars) _h5_write_impl = loc_vars['_h5_write_impl'] f_block = compile_to_numba_ir(_h5_write_impl, {'np': np, 'hpat': hpat}, - self.typingctx, - (self.typemap[dset.name], - self.typemap[arr.name]), - self.typemap, - self.calltypes).blocks.popitem()[1] + self.state.typingctx, + (self.state.typemap[dset.name], + self.state.typemap[arr.name]), + self.state.typemap, + self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [dset, arr]) nodes = f_block.body[:-3] # remove none return return nodes @@ -2186,7 +2199,7 @@ def _handle_sorted_by_key(self, rhs): from numba.targets import quicksort # get key lambda key_lambda_var = dict(rhs.kws)['key'] - key_lambda = guard(get_definition, self.func_ir, key_lambda_var) + key_lambda = guard(get_definition, self.state.func_ir, key_lambda_var) if key_lambda is None or not (isinstance(key_lambda, ir.Expr) and key_lambda.op == 'make_function'): raise ValueError("sorted(): lambda for key not found") @@ -2206,7 +2219,7 @@ def lt(a, b): extra_globals={'_sort_func': numba.njit(sort_func)}) def _get_const_tup(self, tup_var): - tup_def = guard(get_definition, self.func_ir, tup_var) + tup_def = guard(get_definition, self.state.func_ir, tup_var) if isinstance(tup_def, ir.Expr): if tup_def.op == 'binop' and tup_def.fn in ('+', operator.add): return (self._get_const_tup(tup_def.lhs) + self._get_const_tup(tup_def.rhs)) @@ -2215,18 +2228,18 @@ def _get_const_tup(self, tup_var): raise ValueError("constant tuple expected") def _get_dt_index_data(self, dt_var, nodes): - var_def = guard(get_definition, self.func_ir, dt_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, dt_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_datetime_index', 'hpat.hiframes.api'): return var_def.args[0] f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_index_data(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[dt_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[dt_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [dt_var]) nodes += f_block.body[:-2] @@ -2238,8 +2251,8 @@ def _get_series_data(self, series_var, nodes): # e.g. S = init_series(A, None) # XXX assuming init_series() is the only call to create a series # and series._data is never overwritten - var_def = guard(get_definition, self.func_ir, series_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, series_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_series', 'hpat.hiframes.api'): return var_def.args[0] @@ -2248,10 +2261,10 @@ def _get_series_data(self, series_var, nodes): f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_series_data(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[series_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[series_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [series_var]) nodes += f_block.body[:-2] @@ -2260,8 +2273,8 @@ def _get_series_data(self, series_var, nodes): def _get_series_index(self, series_var, nodes): # XXX assuming init_series is the only call to create a series # and series._index is never overwritten - var_def = guard(get_definition, self.func_ir, series_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, series_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if (call_def == ('init_series', 'hpat.hiframes.api') and (len(var_def.args) >= 2 and not self._is_const_none(var_def.args[1]))): @@ -2272,10 +2285,10 @@ def _get_series_index(self, series_var, nodes): f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_series_index(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[series_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[series_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [series_var]) nodes += f_block.body[:-2] @@ -2295,16 +2308,16 @@ def _gen_arange(S): # pragma: no cover return np.arange(n) f_block = compile_to_numba_ir( - _gen_arange, {'np': np}, self.typingctx, - (self.typemap[series_data.name],), - self.typemap, self.calltypes).blocks.popitem()[1] + _gen_arange, {'np': np}, self.state.typingctx, + (self.state.typemap[series_data.name],), + self.state.typemap, self.state.calltypes).blocks.popitem()[1] replace_arg_nodes(f_block, [series_data]) nodes += f_block.body[:-2] return nodes[-1].target def _get_series_name(self, series_var, nodes): - var_def = guard(get_definition, self.func_ir, series_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, series_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if (call_def == ('init_series', 'hpat.hiframes.api') and len(var_def.args) == 3): return var_def.args[2] @@ -2312,28 +2325,28 @@ def _get_series_name(self, series_var, nodes): f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_series_name(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[series_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[series_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [series_var]) nodes += f_block.body[:-2] return nodes[-1].target def _get_timedelta_index_data(self, dt_var, nodes): - var_def = guard(get_definition, self.func_ir, dt_var) - call_def = guard(find_callname, self.func_ir, var_def) + var_def = guard(get_definition, self.state.func_ir, dt_var) + call_def = guard(find_callname, self.state.func_ir, var_def) if call_def == ('init_timedelta_index', 'hpat.hiframes.api'): return var_def.args[0] f_block = compile_to_numba_ir( lambda S: hpat.hiframes.api.get_index_data(S), {'hpat': hpat}, - self.typingctx, - (self.typemap[dt_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[dt_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [dt_var]) nodes += f_block.body[:-2] @@ -2349,7 +2362,7 @@ def _replace_func(self, func, args, const=False, # XXX: inine_closure_call() can't handle defaults properly if pysig is not None: pre_nodes = [] if pre_nodes is None else pre_nodes - scope = next(iter(self.func_ir.blocks.values())).scope + scope = next(iter(self.state.func_ir.blocks.values())).scope loc = scope.loc def normal_handler(index, param, default): @@ -2357,7 +2370,7 @@ def normal_handler(index, param, default): def default_handler(index, param, default): d_var = ir.Var(scope, mk_unique_var('defaults'), loc) - self.typemap[d_var.name] = numba.typeof(default) + self.state.typemap[d_var.name] = numba.typeof(default) node = ir.Assign(ir.Const(default, loc), d_var, loc) pre_nodes.append(node) return d_var @@ -2367,12 +2380,12 @@ def default_handler(index, param, default): pysig, args, kws, normal_handler, default_handler, normal_handler) - arg_typs = tuple(self.typemap[v.name] for v in args) + arg_typs = tuple(self.state.typemap[v.name] for v in args) if const: new_args = [] for i, arg in enumerate(args): - val = guard(find_const, self.func_ir, arg) + val = guard(find_const, self.state.func_ir, arg) if val: new_args.append(types.literal(val)) else: @@ -2381,7 +2394,7 @@ def default_handler(index, param, default): return ReplaceFunc(func, arg_typs, args, glbls, pre_nodes) def _convert_series_calltype(self, call): - sig = self.calltypes[call] + sig = self.state.calltypes[call] if sig is None: return assert isinstance(sig, Signature) @@ -2396,15 +2409,15 @@ def _convert_series_calltype(self, call): # StencilFunc requires kws for typing so sig.args can't be used # reusing sig.args since some types become Const in sig argtyps = new_sig.args[:len(call.args)] - kwtyps = {name: self.typemap[v.name] for name, v in call.kws} + kwtyps = {name: self.state.typemap[v.name] for name, v in call.kws} sig = new_sig - new_sig = self.typemap[call.func.name].get_call_type( - self.typingctx, argtyps, kwtyps) + new_sig = self.state.typemap[call.func.name].get_call_type( + self.state.typingctx, argtyps, kwtyps) # calltypes of things like BoundFunction (array.call) need to # be updated for lowering to work # XXX: new_sig could be None for things like np.int32() - if call in self.calltypes and new_sig is not None: - old_sig = self.calltypes[call] + if call in self.state.calltypes and new_sig is not None: + old_sig = self.state.calltypes[call] # fix types with undefined dtypes in empty_inferred, etc. return_type = _fix_typ_undefs(new_sig.return_type, old_sig.return_type) args = tuple(_fix_typ_undefs(a, b) for a, b in zip(new_sig.args, old_sig.args)) @@ -2413,17 +2426,17 @@ def _convert_series_calltype(self, call): if new_sig is not None: # XXX sometimes new_sig is None for some reason # FIXME e.g. test_series_nlargest_parallel1 np.int32() - self.calltypes.pop(call) - self.calltypes[call] = new_sig + self.state.calltypes.pop(call) + self.state.calltypes[call] = new_sig return def is_bool_arr(self, varname): - typ = self.typemap[varname] + typ = self.state.typemap[varname] return (isinstance(if_series_to_array_type(typ), types.Array) and typ.dtype == types.bool_) def _is_const_none(self, var): - var_def = guard(get_definition, self.func_ir, var) + var_def = guard(get_definition, self.state.func_ir, var) return isinstance(var_def, ir.Const) and var_def.value is None def _handle_hiframes_nodes(self, inst): @@ -2455,7 +2468,7 @@ def _handle_hiframes_nodes(self, inst): else: assert isinstance(inst, hiframes.filter.Filter) use_vars = list(inst.df_in_vars.values()) - if isinstance(self.typemap[inst.bool_arr.name], SeriesType): + if isinstance(self.state.typemap[inst.bool_arr.name], SeriesType): use_vars.append(inst.bool_arr) def_vars = list(inst.df_out_vars.values()) apply_copies_func = hiframes.filter.apply_copies_filter @@ -2469,7 +2482,7 @@ def _update_definitions(self, node_list): loc = ir.Loc("", 0) dumm_block = ir.Block(ir.Scope(None, loc), loc) dumm_block.body = node_list - build_definitions({0: dumm_block}, self.func_ir._definitions) + build_definitions({0: dumm_block}, self.state.func_ir._definitions) return def _convert_series_hiframes_nodes(self, inst, use_vars, def_vars, @@ -2477,26 +2490,26 @@ def _convert_series_hiframes_nodes(self, inst, use_vars, def_vars, # out_nodes = [] varmap = {v.name: self._get_series_data(v, out_nodes) for v in use_vars - if isinstance(self.typemap[v.name], SeriesType)} + if isinstance(self.state.typemap[v.name], SeriesType)} apply_copies_func(inst, varmap, None, None, None, None) out_nodes.append(inst) for v in def_vars: - self.func_ir._definitions[v.name].remove(inst) + self.state.func_ir._definitions[v.name].remove(inst) varmap = {} for v in def_vars: - if not isinstance(self.typemap[v.name], SeriesType): + if not isinstance(self.state.typemap[v.name], SeriesType): continue data_var = ir.Var( v.scope, mk_unique_var(v.name + 'data'), v.loc) - self.typemap[data_var.name] = series_to_array_type(self.typemap[v.name]) + self.state.typemap[data_var.name] = series_to_array_type(self.state.typemap[v.name]) f_block = compile_to_numba_ir( lambda A: hpat.hiframes.api.init_series(A), {'hpat': hpat}, - self.typingctx, - (self.typemap[data_var.name],), - self.typemap, - self.calltypes + self.state.typingctx, + (self.state.typemap[data_var.name],), + self.state.typemap, + self.state.calltypes ).blocks.popitem()[1] replace_arg_nodes(f_block, [data_var]) out_nodes += f_block.body[:-2] diff --git a/hpat/hiframes/hiframes_untyped.py b/hpat/hiframes/hiframes_untyped.py index c313adb59..020fb666e 100644 --- a/hpat/hiframes/hiframes_untyped.py +++ b/hpat/hiframes/hiframes_untyped.py @@ -19,6 +19,7 @@ from numba.inline_closurecall import inline_closure_call from numba.analysis import compute_cfg_from_blocks +from numba.compiler_machinery import FunctionPass, register_pass import hpat from hpat import utils, config @@ -26,7 +27,7 @@ from hpat.io import pio, parquet_pio from hpat.hiframes import filter, join, aggregate, sort from hpat.utils import (get_constant, NOT_CONSTANT, debug_prints, - inline_new_blocks, ReplaceFunc, is_call, is_assign) + inline_new_blocks, ReplaceFunc, is_call, is_assign, update_globals) import hpat.hiframes.api from hpat.str_ext import string_type from hpat.str_arr_ext import string_array_type @@ -102,16 +103,22 @@ def remove_hiframes(rhs, lives, call_list): numba.ir_utils.remove_call_handlers.append(remove_hiframes) -class HiFrames(object): +@register_pass(mutates_CFG=True, analysis_only=False) +class HiFramesPass(FunctionPass): """analyze and transform hiframes calls""" - def __init__(self, func_ir, typingctx, args, _locals, metadata): - self.func_ir = func_ir - self.typingctx = typingctx - self.args = args - self.locals = _locals - self.metadata = metadata - ir_utils._max_label = max(func_ir.blocks.keys()) + _name = "hi_frames_pass" + + def __init__(self): + pass + + def run_pass(self, state): + return HiFramesPassImpl(state).run_pass() + + +class HiFramesPassImpl(object): + + def __init__(self, state): # replace inst variables as determined previously during the pass # currently use to keep lhs of Arg nodes intact self.replace_var_dict = {} @@ -123,19 +130,24 @@ def __init__(self, func_ir, typingctx, args, _locals, metadata): self.arrow_tables = {} self.reverse_copies = {} + + self.state = state + + def run_pass(self): + ir_utils._max_label = max(self.state.func_ir.blocks.keys()) + self.pq_handler = ParquetHandler( - func_ir, typingctx, args, _locals, self.reverse_copies) - self.h5_handler = pio.PIO(self.func_ir, _locals, self.reverse_copies) + self.state.func_ir, self.state.typingctx, self.state.args, self.state.locals, self.reverse_copies) + self.h5_handler = pio.PIO(self.state.func_ir, self.state.locals, self.reverse_copies) - def run(self): # FIXME: see why this breaks test_kmeans - # remove_dels(self.func_ir.blocks) - dprint_func_ir(self.func_ir, "starting hiframes") + # remove_dels(self.state.func_ir.blocks) + dprint_func_ir(self.state.func_ir, "starting hiframes") self._handle_metadata() - blocks = self.func_ir.blocks + blocks = self.state.func_ir.blocks # call build definition since rewrite pass doesn't update definitions # e.g. getitem to static_getitem in test_column_list_select2 - self.func_ir._definitions = build_definitions(blocks) + self.state.func_ir._definitions = build_definitions(blocks) # topo_order necessary since df vars need to be found before use topo_order = find_topo_order(blocks) work_list = list((l, blocks[l]) for l in reversed(topo_order)) @@ -158,7 +170,7 @@ def run(self): cfg = compute_cfg_from_blocks(blocks) out_nodes = self._run_df_set_column(inst, label, cfg) elif isinstance(inst, ir.Assign): - self.func_ir._definitions[inst.target.name].remove(inst.value) + self.state.func_ir._definitions[inst.target.name].remove(inst.value) out_nodes = self._run_assign(inst, label) elif isinstance(inst, ir.Return): out_nodes = self._run_return(inst) @@ -179,7 +191,8 @@ def run(self): ir.Var(block.scope, "dummy", inst.loc), rp_func.args, (), inst.loc) block.body = new_body + block.body[i:] - inline_closure_call(self.func_ir, rp_func.glbls, + update_globals(rp_func.func, rp_func.glbls) + inline_closure_call(self.state.func_ir, rp_func.glbls, block, len(new_body), rp_func.func, work_list=work_list) replaced = True break @@ -188,38 +201,39 @@ def run(self): # TODO: insert new blocks in current spot of work_list # instead of append? # TODO: rename variables, fix scope/loc - inline_new_blocks(self.func_ir, block, len(new_body), out_nodes, work_list) + inline_new_blocks(self.state.func_ir, block, len(new_body), out_nodes, work_list) replaced = True break if not replaced: blocks[label].body = new_body - self.func_ir.blocks = ir_utils.simplify_CFG(self.func_ir.blocks) - # self.func_ir._definitions = build_definitions(blocks) + self.state.func_ir.blocks = ir_utils.simplify_CFG(self.state.func_ir.blocks) + # self.state.func_ir._definitions = build_definitions(blocks) # XXX: remove dead here fixes h5 slice issue # iterative remove dead to make sure all extra code (e.g. df vars) is removed - # while remove_dead(blocks, self.func_ir.arg_names, self.func_ir): + # while remove_dead(blocks, self.state.func_ir.arg_names, self.state.func_ir): # pass - self.func_ir._definitions = build_definitions(blocks) - dprint_func_ir(self.func_ir, "after hiframes") + self.state.func_ir._definitions = build_definitions(blocks) + dprint_func_ir(self.state.func_ir, "after hiframes") if debug_prints(): # pragma: no cover print("df_vars: ", self.df_vars) - return + + return True def _replace_vars(self, inst): # variable replacement can affect definitions so handling assignment # values specifically if is_assign(inst): lhs = inst.target.name - self.func_ir._definitions[lhs].remove(inst.value) + self.state.func_ir._definitions[lhs].remove(inst.value) ir_utils.replace_vars_stmt(inst, self.replace_var_dict) if is_assign(inst): - self.func_ir._definitions[lhs].append(inst.value) + self.state.func_ir._definitions[lhs].append(inst.value) # if lhs changed, TODO: test if inst.target.name != lhs: - self.func_ir._definitions[inst.target.name] = self.func_ir._definitions[lhs] + self.state.func_ir._definitions[inst.target.name] = self.state.func_ir._definitions[lhs] def _run_assign(self, assign, label): lhs = assign.target.name @@ -239,37 +253,37 @@ def _run_assign(self, assign, label): # HACK: delete pd.DataFrame({}) nodes to avoid typing errors # TODO: remove when dictionaries are implemented and typing works if rhs.op == 'getattr': - val_def = guard(get_definition, self.func_ir, rhs.value) + val_def = guard(get_definition, self.state.func_ir, rhs.value) if (isinstance(val_def, ir.Global) and val_def.value == pd and rhs.attr in ('DataFrame', 'read_csv', 'read_parquet', 'to_numeric')): # TODO: implement to_numeric in typed pass? # put back the definition removed earlier but remove node # enables function matching without node in IR - self.func_ir._definitions[lhs].append(rhs) + self.state.func_ir._definitions[lhs].append(rhs) return [] if rhs.op == 'getattr': - val_def = guard(get_definition, self.func_ir, rhs.value) + val_def = guard(get_definition, self.state.func_ir, rhs.value) if (isinstance(val_def, ir.Global) and val_def.value == np and rhs.attr == 'fromfile'): # put back the definition removed earlier but remove node - self.func_ir._definitions[lhs].append(rhs) + self.state.func_ir._definitions[lhs].append(rhs) return [] # HACK: delete pyarrow.parquet.read_table() to avoid typing errors if rhs.op == 'getattr' and rhs.attr == 'read_table': import pyarrow.parquet as pq - val_def = guard(get_definition, self.func_ir, rhs.value) + val_def = guard(get_definition, self.state.func_ir, rhs.value) if isinstance(val_def, ir.Global) and val_def.value == pq: # put back the definition removed earlier but remove node - self.func_ir._definitions[lhs].append(rhs) + self.state.func_ir._definitions[lhs].append(rhs) return [] if (rhs.op == 'getattr' and rhs.value.name in self.arrow_tables and rhs.attr == 'to_pandas'): # put back the definition removed earlier but remove node - self.func_ir._definitions[lhs].append(rhs) + self.state.func_ir._definitions[lhs].append(rhs) return [] # if rhs.op in ('build_list', 'build_tuple'): TODO: test tuple @@ -282,7 +296,7 @@ def _run_assign(self, assign, label): # XXX: when constants are used, all the uses of the list object # have to be checked since lists are mutable try: - vals = tuple(find_const(self.func_ir, v) for v in rhs.items) + vals = tuple(find_const(self.state.func_ir, v) for v in rhs.items) # a = ['A', 'B'] -> # tmp = ['A', 'B'] # a = add_consts_to_type(tmp, 'A', 'B') @@ -303,17 +317,17 @@ def _run_assign(self, assign, label): if rhs.op == 'make_function': # HACK make globals availabe for typing in series.map() - rhs.globals = self.func_ir.func_id.func.__globals__ + rhs.globals = self.state.func_ir.func_id.func.__globals__ # pass pivot values to df.pivot_table() calls using a meta # variable passed as argument. The meta variable's type # is set to MetaType with pivot values baked in. pivot_key = lhs + ":pivot" - if pivot_key in self.locals: - pivot_values = self.locals.pop(pivot_key) + if pivot_key in self.state.locals: + pivot_values = self.state.locals.pop(pivot_key) # put back the definition removed earlier - self.func_ir._definitions[lhs].append(rhs) - pivot_call = guard(get_definition, self.func_ir, lhs) + self.state.func_ir._definitions[lhs].append(rhs) + pivot_call = guard(get_definition, self.state.func_ir, lhs) assert pivot_call is not None meta_var = ir.Var( assign.target.scope, mk_unique_var('pivot_meta'), rhs.loc) @@ -322,7 +336,7 @@ def _run_assign(self, assign, label): self._working_body.insert(0, meta_assign) pivot_call.kws = list(pivot_call.kws) pivot_call.kws.append(('_pivot_values', meta_var)) - self.locals[meta_var.name] = hpat.hiframes.api.MetaType(pivot_values) + self.state.locals[meta_var.name] = hpat.hiframes.api.MetaType(pivot_values) # handle copies lhs = f if isinstance(rhs, ir.Var) and rhs.name in self.df_vars: @@ -332,7 +346,7 @@ def _run_assign(self, assign, label): if isinstance(rhs, ir.Var) and rhs.name in self.arrow_tables: self.arrow_tables[lhs] = self.arrow_tables[rhs.name] # enables function matching without node in IR - self.func_ir._definitions[lhs].append(rhs) + self.state.func_ir._definitions[lhs].append(rhs) return [] return [assign] @@ -344,10 +358,10 @@ def _run_call(self, assign, label): func_name = "" func_mod = "" - fdef = guard(find_callname, self.func_ir, rhs) + fdef = guard(find_callname, self.state.func_ir, rhs) if fdef is None: # could be make_function from list comprehension which is ok - func_def = guard(get_definition, self.func_ir, rhs.func) + func_def = guard(get_definition, self.state.func_ir, rhs.func) if isinstance(func_def, ir.Expr) and func_def.op == 'make_function': return [assign] warnings.warn( @@ -417,7 +431,7 @@ def _run_call(self, assign, label): return hpat.io.np_io._handle_np_fromfile(assign, lhs, rhs) if fdef == ('read_xenon', 'hpat.xenon_ext'): - col_items, nodes = hpat.xenon_ext._handle_read(assign, lhs, rhs, self.func_ir) + col_items, nodes = hpat.xenon_ext._handle_read(assign, lhs, rhs, self.state.func_ir) df_nodes, col_map = self._process_df_build_map(col_items) self._create_df(lhs.name, col_map, label) nodes += df_nodes @@ -464,11 +478,11 @@ def _handle_df_isin(self, lhs, rhs, df_var, label): if cname in arg_df_map: other_colmap[cname] = arg_df_map[cname] else: - other_def = guard(get_definition, self.func_ir, other) + other_def = guard(get_definition, self.state.func_ir, other) # dict case if isinstance(other_def, ir.Expr) and other_def.op == 'build_map': for c, v in other_def.items: - cname = guard(find_const, self.func_ir, c) + cname = guard(find_const, self.state.func_ir, c) if not isinstance(cname, str): raise ValueError("dictionary argument to isin() should have constant keys") other_colmap[cname] = v @@ -516,7 +530,7 @@ def _handle_df_append(self, lhs, rhs, df_var, label): if self._is_df_var(other): return self._handle_concat_df(lhs, [df_var, other], label) # list of dfs - df_list = guard(get_definition, self.func_ir, other) + df_list = guard(get_definition, self.state.func_ir, other) if len(df_list.items) > 0 and self._is_df_var(df_list.items[0]): return self._handle_concat_df(lhs, [df_var] + df_list.items, label) raise ValueError("invalid df.append() input. Only dataframe and list" @@ -539,7 +553,7 @@ def _fillna_func(A, val, inplace): return A.fillna(val, inplace=inplace) # create output df if not inplace if (inplace_var.name == inplace_default.name - or guard(find_const, self.func_ir, inplace_var) == False): + or guard(find_const, self.state.func_ir, inplace_var) == False): self._create_df(lhs.name, out_col_map, label) return nodes @@ -558,7 +572,7 @@ def _handle_df_dropna(self, lhs, rhs, df_var, label): func_text += " ({},) = hpat.hiframes.api.dropna(({},), inplace)\n".format( out_names, arg_names) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _dropna_imp = loc_vars['_dropna_imp'] f_block = compile_to_numba_ir(_dropna_imp, {'hpat': hpat}).blocks.popitem()[1] @@ -572,7 +586,7 @@ def _handle_df_dropna(self, lhs, rhs, df_var, label): # create output df if not inplace if (inplace_var.name == inplace_default.name - or guard(find_const, self.func_ir, inplace_var) == False): + or guard(find_const, self.state.func_ir, inplace_var) == False): self._create_df(lhs.name, out_col_map, label) else: # assign back to column vars for inplace case @@ -589,7 +603,7 @@ def _handle_df_drop(self, assign, lhs, rhs, df_var): """ kws = dict(rhs.kws) inplace_var = self._get_arg('drop', rhs.args, kws, 5, 'inplace', '') - inplace = guard(find_const, self.func_ir, inplace_var) + inplace = guard(find_const, self.state.func_ir, inplace_var) if inplace is not None and inplace: # TODO: make sure call post dominates df_var definition or df_var # is not used in other code paths @@ -625,7 +639,7 @@ def _handle_df_drop(self, assign, lhs, rhs, df_var): labels_var = self._get_arg('drop', rhs.args, kws, 0, 'labels', '') axis_var = self._get_arg('drop', rhs.args, kws, 1, 'axis', '') labels = self._get_str_or_list(labels_var, default='') - axis = guard(find_const, self.func_ir, axis_var) + axis = guard(find_const, self.state.func_ir, axis_var) if labels != '' and axis is not None: if axis != 1: @@ -638,11 +652,11 @@ def _handle_df_drop(self, assign, lhs, rhs, df_var): columns = self._get_str_or_list(columns_var, err_msg=err_msg) inplace_var = self._get_arg('drop', rhs.args, kws, 5, 'inplace', '') - inplace = guard(find_const, self.func_ir, inplace_var) + inplace = guard(find_const, self.state.func_ir, inplace_var) if inplace is not None and inplace: df_label = self.df_labels[df_var.name] - cfg = compute_cfg_from_blocks(self.func_ir.blocks) + cfg = compute_cfg_from_blocks(self.state.func_ir.blocks) # dropping columns inplace possible only when it dominates the df # creation to keep schema consistent if label not in cfg.backbone() and label not in cfg.post_dominators()[df_label]: @@ -682,7 +696,7 @@ def _handle_pd_DataFrame(self, assign, lhs, rhs, label): "data argument in pd.DataFrame() expected") data = rhs.args[0] - arg_def = guard(get_definition, self.func_ir, data) + arg_def = guard(get_definition, self.state.func_ir, data) if (not isinstance(arg_def, ir.Expr) or arg_def.op != 'build_map'): # pragma: no cover raise ValueError( @@ -700,7 +714,7 @@ def _handle_pd_DataFrame(self, assign, lhs, rhs, label): func_text += " return hpat.hiframes.pd_dataframe_ext.init_dataframe({}, index, {})\n".format( data_args, col_args) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _init_df = loc_vars['_init_df'] # TODO: support index var @@ -751,7 +765,7 @@ def _handle_pd_read_csv(self, assign, lhs, rhs, label): col_names = self._get_str_or_list(names_var, default=0) if dtype_var is '': # infer column names and types from constant filename - fname_const = guard(find_const, self.func_ir, fname) + fname_const = guard(find_const, self.state.func_ir, fname) if fname_const is None: raise ValueError("pd.read_csv() requires explicit type" "annotation using 'dtype' if filename is not constant") @@ -770,7 +784,7 @@ def _handle_pd_read_csv(self, assign, lhs, rhs, label): col_names = cols dtype_map = {c: d for c, d in zip(col_names, dtypes)} else: - dtype_map = guard(get_definition, self.func_ir, dtype_var) + dtype_map = guard(get_definition, self.state.func_ir, dtype_var) if (not isinstance(dtype_map, ir.Expr) or dtype_map.op != 'build_map'): # pragma: no cover # try single type for all columns case @@ -779,7 +793,7 @@ def _handle_pd_read_csv(self, assign, lhs, rhs, label): new_dtype_map = {} for n_var, t_var in dtype_map.items: # find constant column name - c = guard(find_const, self.func_ir, n_var) + c = guard(find_const, self.state.func_ir, n_var) if c is None: # pragma: no cover raise ValueError("dtype column names should be constant") new_dtype_map[c] = self._get_const_dtype(t_var) @@ -818,7 +832,7 @@ def _handle_pd_read_csv(self, assign, lhs, rhs, label): func_text += " return hpat.hiframes.pd_dataframe_ext.init_dataframe({}, None, {})\n".format( data_args, ", ".join("'{}'".format(c) for c in columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _init_df = loc_vars['_init_df'] f_block = compile_to_numba_ir( @@ -851,7 +865,7 @@ def _get_csv_col_info(self, dtype_map, date_cols, col_names, lhs): return columns, data_arrs, out_types def _get_const_dtype(self, dtype_var): - dtype_def = guard(get_definition, self.func_ir, dtype_var) + dtype_def = guard(get_definition, self.state.func_ir, dtype_var) if isinstance(dtype_def, ir.Const) and isinstance(dtype_def.value, str): typ_name = dtype_def.value if typ_name == 'str': @@ -867,7 +881,7 @@ def _get_const_dtype(self, dtype_var): return string_array_type # categorical case if isinstance(dtype_def, ir.Expr) and dtype_def.op == 'call': - if (not guard(find_callname, self.func_ir, dtype_def) + if (not guard(find_callname, self.state.func_ir, dtype_def) == ('category', 'pandas.core.dtypes.dtypes')): raise ValueError("pd.read_csv() invalid dtype " "(built using a call but not Categorical)") @@ -879,7 +893,7 @@ def _get_const_dtype(self, dtype_var): return CategoricalArray(typ) if not isinstance(dtype_def, ir.Expr) or dtype_def.op != 'getattr': raise ValueError("pd.read_csv() invalid dtype") - glob_def = guard(get_definition, self.func_ir, dtype_def.value) + glob_def = guard(get_definition, self.state.func_ir, dtype_def.value) if not isinstance(glob_def, ir.Global) or glob_def.value != np: raise ValueError("pd.read_csv() invalid dtype") # TODO: extend to other types like string and date, check error @@ -897,11 +911,11 @@ def _handle_pd_Series(self, assign, lhs, rhs): data = self._get_arg('pd.Series', rhs.args, kws, 0, 'data') # match flatmap pd.Series(list(itertools.chain(*A))) and flatten - data_def = guard(get_definition, self.func_ir, data) - if (is_call(data_def) and guard(find_callname, self.func_ir, data_def) + data_def = guard(get_definition, self.state.func_ir, data) + if (is_call(data_def) and guard(find_callname, self.state.func_ir, data_def) == ('list', 'builtins') and len(data_def.args) == 1): - arg_def = guard(get_definition, self.func_ir, data_def.args[0]) - if (is_call(arg_def) and guard(find_callname, self.func_ir, + arg_def = guard(get_definition, self.state.func_ir, data_def.args[0]) + if (is_call(arg_def) and guard(find_callname, self.state.func_ir, arg_def) == ('chain', 'itertools')): in_data = arg_def.vararg arg_def.vararg = None # avoid typing error @@ -921,13 +935,13 @@ def _handle_pd_to_numeric(self, assign, lhs, rhs): has to be specified in locals and applied """ kws = dict(rhs.kws) - if 'errors' not in kws or guard(find_const, self.func_ir, kws['errors']) != 'coerce': + if 'errors' not in kws or guard(find_const, self.state.func_ir, kws['errors']) != 'coerce': raise ValueError("pd.to_numeric() only supports errors='coerce'") - if lhs.name not in self.reverse_copies or (self.reverse_copies[lhs.name]) not in self.locals: + if lhs.name not in self.reverse_copies or (self.reverse_copies[lhs.name]) not in self.state.locals: raise ValueError("pd.to_numeric() requires annotation of output type") - typ = self.locals.pop(self.reverse_copies[lhs.name]) + typ = self.state.locals.pop(self.reverse_copies[lhs.name]) dtype = numba.numpy_support.as_dtype(typ.dtype) arg = rhs.args[0] @@ -939,7 +953,7 @@ def _handle_pq_read_table(self, assign, lhs, rhs): if len(rhs.args) != 1: # pragma: no cover raise ValueError("Invalid read_table() arguments") # put back the definition removed earlier but remove node - self.func_ir._definitions[lhs.name].append(rhs) + self.state.func_ir._definitions[lhs.name].append(rhs) self.arrow_tables[lhs.name] = rhs.args[0] return [] @@ -956,7 +970,7 @@ def _gen_parquet_read(self, fname, lhs, label): func_text += " return hpat.hiframes.pd_dataframe_ext.init_dataframe({}, None, {})\n".format( data_args, ", ".join("'{}'".format(c) for c in columns)) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _init_df = loc_vars['_init_df'] return self._replace_func(_init_df, data_arrs, pre_nodes=nodes) @@ -971,7 +985,7 @@ def _handle_concat(self, assign, lhs, rhs, label): kws = dict(rhs.kws) objs_arg = self._get_arg('concat', rhs.args, kws, 0, 'objs') - df_list = guard(get_definition, self.func_ir, objs_arg) + df_list = guard(get_definition, self.state.func_ir, objs_arg) if not isinstance(df_list, ir.Expr) or not (df_list.op in ['build_tuple', 'build_list']): raise ValueError("pd.concat input should be constant list or tuple") @@ -1007,7 +1021,7 @@ def gen_nan_func(A): return np.full(len(A), np.nan) func_text += " return hpat.hiframes.api.init_series(hpat.hiframes.api.concat(({})))\n".format( arg_names) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _concat_imp = loc_vars['_concat_imp'] done_cols = {} @@ -1071,8 +1085,8 @@ def f(arr): # pragma: no cover return nodes, new_list def _fix_df_list_of_array(self, col_arr): - list_call = guard(get_definition, self.func_ir, col_arr) - if guard(find_callname, self.func_ir, list_call) == ('list', 'builtins'): + list_call = guard(get_definition, self.state.func_ir, col_arr) + if guard(find_callname, self.state.func_ir, list_call) == ('list', 'builtins'): return list_call.args[0] return col_arr @@ -1084,7 +1098,7 @@ def _process_df_build_map(self, items_list): if isinstance(col_var, str): col_name = col_var else: - col_name = get_constant(self.func_ir, col_var) + col_name = get_constant(self.state.func_ir, col_var) if col_name is NOT_CONSTANT: # pragma: no cover raise ValueError( "data frame column names should be constant") @@ -1103,7 +1117,7 @@ def f(arr): # pragma: no cover def _get_func_output_typ(self, col_var, func, wrapper_func, label): # stich together all blocks before the current block for type inference # XXX: does control flow affect type inference in Numba? - dummy_ir = self.func_ir.copy() + dummy_ir = self.state.func_ir.copy() dummy_ir.blocks[label].body.append(ir.Return(0, col_var.loc)) topo_order = find_topo_order(dummy_ir.blocks) all_body = [] @@ -1117,7 +1131,7 @@ def _get_func_output_typ(self, col_var, func, wrapper_func, label): dummy_ir.blocks = {0: ir.Block(col_var.scope, col_var.loc)} dummy_ir.blocks[0].body = all_body - _globals = self.func_ir.func_id.func.__globals__ + _globals = self.state.func_ir.func_id.func.__globals__ _globals.update({'hpat': hpat, 'numba': numba, 'np': np}) f_ir = compile_to_numba_ir(wrapper_func, {'hpat': hpat}) # fix definitions to enable finding sentinel @@ -1129,6 +1143,7 @@ def _get_func_output_typ(self, col_var, func, wrapper_func, label): and stmt.value.op == 'call'): fdef = guard(get_definition, f_ir, stmt.value.func) if isinstance(fdef, ir.Global) and fdef.name == 'map_func': + update_globals(func, _globals) inline_closure_call(f_ir, _globals, f_ir.blocks[first_label], i, func) break @@ -1150,26 +1165,26 @@ def _get_func_output_typ(self, col_var, func, wrapper_func, label): # run type inference on the dummy IR warnings = numba.errors.WarningsFixer(numba.errors.NumbaWarning) - infer = numba.typeinfer.TypeInferer(self.typingctx, dummy_ir, warnings) - for index, (name, ty) in enumerate(zip(dummy_ir.arg_names, self.args)): + infer = numba.typeinfer.TypeInferer(self.state.typingctx, dummy_ir, warnings) + for index, (name, ty) in enumerate(zip(dummy_ir.arg_names, self.state.args)): infer.seed_argument(name, index, ty) infer.build_constraint() infer.propagate() out_tp = infer.typevars[output_var.name].getone() - # typemap, restype, calltypes = numba.compiler.type_inference_stage(self.typingctx, dummy_ir, self.args, None) + # typemap, restype, calltypes = numba.typed_passes.type_inference_stage(self.state.typingctx, dummy_ir, self.state.args, None) return out_tp def _is_df_obj_call(self, call_var, obj_name): """determines whether variable is coming from groupby() or groupby()[], rolling(), rolling()[] """ - var_def = guard(get_definition, self.func_ir, call_var) + var_def = guard(get_definition, self.state.func_ir, call_var) # groupby()['B'] case if (isinstance(var_def, ir.Expr) and var_def.op in ['getitem', 'static_getitem']): return self._is_df_obj_call(var_def.value, obj_name) # groupby() called on column or df - call_def = guard(find_callname, self.func_ir, var_def) + call_def = guard(find_callname, self.state.func_ir, var_def) if (call_def is not None and call_def[0] == obj_name and isinstance(call_def[1], ir.Var) and self._is_df_var(call_def[1])): @@ -1184,7 +1199,7 @@ def _handle_df_pivot_table(self, lhs, rhs, df_var, label): columns_arg = self._get_str_arg('pivot_table', rhs.args, kws, 2, 'columns') agg_func_arg = self._get_str_arg('pivot_table', rhs.args, kws, 3, 'aggfunc', 'mean') - agg_func = get_agg_func(self.func_ir, agg_func_arg, rhs) + agg_func = get_agg_func(self.state.func_ir, agg_func_arg, rhs) in_vars = {values_arg: self.df_vars[df_var.name][values_arg]} # get output type @@ -1216,19 +1231,19 @@ def to_arr(a, _agg_f): return nodes def _get_pivot_values(self, varname): - if varname not in self.reverse_copies or (self.reverse_copies[varname] + ':pivot') not in self.locals: + if varname not in self.reverse_copies or (self.reverse_copies[varname] + ':pivot') not in self.state.locals: raise ValueError("pivot_table() requires annotation of pivot values") new_name = self.reverse_copies[varname] - values = self.locals.pop(new_name + ":pivot") + values = self.state.locals.pop(new_name + ":pivot") return values def _get_str_arg(self, f_name, args, kws, arg_no, arg_name, default=None, err_msg=None): arg = None if len(args) > arg_no: - arg = guard(find_const, self.func_ir, args[arg_no]) + arg = guard(find_const, self.state.func_ir, args[arg_no]) elif arg_name in kws: - arg = guard(find_const, self.func_ir, kws[arg_name]) + arg = guard(find_const, self.state.func_ir, kws[arg_name]) if arg is None: if default is not None: @@ -1349,7 +1364,7 @@ def _handle_aggregate(self, lhs, rhs, obj_var, func_name, label): return nodes def _handle_agg_func(self, in_vars, out_colnames, func_name, lhs, rhs): - agg_func = get_agg_func(self.func_ir, func_name, rhs) + agg_func = get_agg_func(self.state.func_ir, func_name, rhs) out_tp_vars = {} # hpat.jit() instead of numba.njit() to handle str arrs etc @@ -1371,12 +1386,12 @@ def to_arr(a, _agg_f): def _get_agg_obj_args(self, agg_var): # find groupby key and as_index - groubpy_call = guard(get_definition, self.func_ir, agg_var) + groubpy_call = guard(get_definition, self.state.func_ir, agg_var) assert isinstance(groubpy_call, ir.Expr) and groubpy_call.op == 'call' kws = dict(groubpy_call.kws) as_index = True if 'as_index' in kws: - as_index = guard(find_const, self.func_ir, kws['as_index']) + as_index = guard(find_const, self.state.func_ir, kws['as_index']) if as_index is None: raise ValueError( "groupby as_index argument should be constant") @@ -1395,34 +1410,34 @@ def _get_agg_obj_args(self, agg_var): def _get_str_or_list(self, by_arg, list_only=False, default=None, err_msg=None, typ=None): typ = str if typ is None else typ - by_arg_def = guard(find_build_sequence, self.func_ir, by_arg) + by_arg_def = guard(find_build_sequence, self.state.func_ir, by_arg) if by_arg_def is None: # try add_consts_to_type - by_arg_call = guard(get_definition, self.func_ir, by_arg) - if guard(find_callname, self.func_ir, by_arg_call) == ('add_consts_to_type', 'hpat.hiframes.api'): - by_arg_def = guard(find_build_sequence, self.func_ir, by_arg_call.args[0]) + by_arg_call = guard(get_definition, self.state.func_ir, by_arg) + if guard(find_callname, self.state.func_ir, by_arg_call) == ('add_consts_to_type', 'hpat.hiframes.api'): + by_arg_def = guard(find_build_sequence, self.state.func_ir, by_arg_call.args[0]) if by_arg_def is None: # try dict.keys() - by_arg_call = guard(get_definition, self.func_ir, by_arg) - call_name = guard(find_callname, self.func_ir, by_arg_call) + by_arg_call = guard(get_definition, self.state.func_ir, by_arg) + call_name = guard(find_callname, self.state.func_ir, by_arg_call) if (call_name is not None and len(call_name) == 2 and call_name[0] == 'keys' and isinstance(call_name[1], ir.Var)): - var_def = guard(get_definition, self.func_ir, call_name[1]) + var_def = guard(get_definition, self.state.func_ir, call_name[1]) if isinstance(var_def, ir.Expr) and var_def.op == 'build_map': by_arg_def = [v[0] for v in var_def.items], 'build_map' # HACK replace dict.keys getattr to avoid typing errors keys_getattr = guard( - get_definition, self.func_ir, by_arg_call.func) + get_definition, self.state.func_ir, by_arg_call.func) assert isinstance( keys_getattr, ir.Expr) and keys_getattr.attr == 'keys' keys_getattr.attr = 'copy' if by_arg_def is None: # try single key column - by_arg_def = guard(find_const, self.func_ir, by_arg) + by_arg_def = guard(find_const, self.state.func_ir, by_arg) if by_arg_def is None: if default is not None: return default @@ -1433,7 +1448,7 @@ def _get_str_or_list(self, by_arg, list_only=False, default=None, err_msg=None, if default is not None: return default raise ValueError(err_msg) - key_colnames = [guard(find_const, self.func_ir, v) for v in by_arg_def[0]] + key_colnames = [guard(find_const, self.state.func_ir, v) for v in by_arg_def[0]] if any(not isinstance(v, typ) for v in key_colnames): if default is not None: return default @@ -1444,23 +1459,23 @@ def _get_df_obj_select(self, obj_var, obj_name): """analyze selection of columns in after groupby() or rolling() e.g. groupby('A')['B'], groupby('A')['B', 'C'], groupby('A') """ - select_def = guard(get_definition, self.func_ir, obj_var) + select_def = guard(get_definition, self.state.func_ir, obj_var) out_colnames = None explicit_select = False if isinstance(select_def, ir.Expr) and select_def.op in ('getitem', 'static_getitem'): obj_var = select_def.value out_colnames = (select_def.index if select_def.op == 'static_getitem' - else guard(find_const, self.func_ir, select_def.index)) + else guard(find_const, self.state.func_ir, select_def.index)) if not isinstance(out_colnames, (str, tuple)): raise ValueError("{} output column names should be constant".format(obj_name)) if isinstance(out_colnames, str): out_colnames = [out_colnames] explicit_select = True - obj_call = guard(get_definition, self.func_ir, obj_var) + obj_call = guard(get_definition, self.state.func_ir, obj_var) # find dataframe - call_def = guard(find_callname, self.func_ir, obj_call) + call_def = guard(find_callname, self.state.func_ir, obj_call) assert (call_def is not None and call_def[0] == obj_name and isinstance(call_def[1], ir.Var) and self._is_df_var(call_def[1])) @@ -1478,8 +1493,8 @@ def _handle_rolling(self, lhs, rhs, obj_var, func_name, label): nodes = [] # find selected output columns df_var, out_colnames, explicit_select, obj_var = self._get_df_obj_select(obj_var, 'rolling') - rolling_call = guard(get_definition, self.func_ir, obj_var) - window, center, on = get_rolling_setup_args(self.func_ir, rolling_call, False) + rolling_call = guard(get_definition, self.state.func_ir, obj_var) + window, center, on = get_rolling_setup_args(self.state.func_ir, rolling_call, False) on_arr = self.df_vars[df_var.name][on] if on is not None else None if not isinstance(center, ir.Var): center_var = ir.Var(lhs.scope, mk_unique_var("center"), lhs.loc) @@ -1630,57 +1645,57 @@ def f(arr): # pragma: no cover def _handle_metadata(self): """remove distributed input annotation from locals and add to metadata """ - if 'distributed' not in self.metadata: + if 'distributed' not in self.state.metadata: # TODO: keep updated in variable renaming? - self.metadata['distributed'] = self.locals.pop( + self.state.metadata['distributed'] = self.state.locals.pop( '##distributed', set()) - if 'threaded' not in self.metadata: - self.metadata['threaded'] = self.locals.pop('##threaded', set()) + if 'threaded' not in self.state.metadata: + self.state.metadata['threaded'] = self.state.locals.pop('##threaded', set()) # handle old input flags # e.g. {"A:input": "distributed"} -> "A" dist_inputs = {var_name.split(":")[0] - for (var_name, flag) in self.locals.items() + for (var_name, flag) in self.state.locals.items() if var_name.endswith(":input") and flag == 'distributed'} thread_inputs = {var_name.split(":")[0] - for (var_name, flag) in self.locals.items() + for (var_name, flag) in self.state.locals.items() if var_name.endswith(":input") and flag == 'threaded'} # check inputs to be in actuall args for arg_name in dist_inputs | thread_inputs: - if arg_name not in self.func_ir.arg_names: + if arg_name not in self.state.func_ir.arg_names: raise ValueError( "distributed input {} not found in arguments".format( arg_name)) - self.locals.pop(arg_name + ":input") + self.state.locals.pop(arg_name + ":input") - self.metadata['distributed'] |= dist_inputs - self.metadata['threaded'] |= thread_inputs + self.state.metadata['distributed'] |= dist_inputs + self.state.metadata['threaded'] |= thread_inputs # handle old return flags # e.g. {"A:return":"distributed"} -> "A" flagged_returns = {var_name.split(":")[0]: flag - for (var_name, flag) in self.locals.items() + for (var_name, flag) in self.state.locals.items() if var_name.endswith(":return")} for v, flag in flagged_returns.items(): if flag == 'distributed': - self.metadata['distributed'].add(v) + self.state.metadata['distributed'].add(v) elif flag == 'threaded': - self.metadata['threaded'].add(v) + self.state.metadata['threaded'].add(v) - self.locals.pop(v + ":return") + self.state.locals.pop(v + ":return") return def _run_return(self, ret_node): # TODO: handle distributed analysis, requires handling variable name # change in simplify() and replace_var_names() - flagged_vars = self.metadata['distributed'] | self.metadata['threaded'] + flagged_vars = self.state.metadata['distributed'] | self.state.metadata['threaded'] nodes = [ret_node] - cast = guard(get_definition, self.func_ir, ret_node.value) + cast = guard(get_definition, self.state.func_ir, ret_node.value) assert cast is not None, "return cast not found" assert isinstance(cast, ir.Expr) and cast.op == 'cast' scope = cast.value.scope @@ -1689,7 +1704,7 @@ def _run_return(self, ret_node): ret_name = cast.value.name.split('.')[0] if ret_name in flagged_vars: - flag = ('distributed' if ret_name in self.metadata['distributed'] + flag = ('distributed' if ret_name in self.state.metadata['distributed'] else 'threaded') nodes = self._gen_replace_dist_return(cast.value, flag) new_arr = nodes[-1].target @@ -1704,7 +1719,7 @@ def _run_return(self, ret_node): if len(flagged_vars) == 0: return nodes - cast_def = guard(get_definition, self.func_ir, cast.value) + cast_def = guard(get_definition, self.state.func_ir, cast.value) if (cast_def is not None and isinstance(cast_def, ir.Expr) and cast_def.op == 'build_tuple'): nodes = [] @@ -1712,7 +1727,7 @@ def _run_return(self, ret_node): for v in cast_def.items: vname = v.name.split('.')[0] if vname in flagged_vars: - flag = ('distributed' if vname in self.metadata['distributed'] + flag = ('distributed' if vname in self.state.metadata['distributed'] else 'threaded') nodes += self._gen_replace_dist_return(v, flag) new_var_list.append(nodes[-1].target) @@ -1847,7 +1862,7 @@ def _update_definitions(self, node_list): loc = ir.Loc("", 0) dumm_block = ir.Block(ir.Scope(None, loc), loc) dumm_block.body = node_list - build_definitions({0: dumm_block}, self.func_ir._definitions) + build_definitions({0: dumm_block}, self.state.func_ir._definitions) return diff --git a/hpat/hiframes/join.py b/hpat/hiframes/join.py index 9804a0958..0892e88cc 100644 --- a/hpat/hiframes/join.py +++ b/hpat/hiframes/join.py @@ -426,7 +426,7 @@ def join_distributed_run(join_node, array_dists, typemap, calltypes, typingctx, func_text += " {} = right_{}\n".format(out_names[i + 2 * n_keys + len(left_other_names)], i) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) join_impl = loc_vars['f'] # print(func_text) diff --git a/hpat/hiframes/pd_dataframe_ext.py b/hpat/hiframes/pd_dataframe_ext.py index cd1d7f1e5..b86e1ae47 100644 --- a/hpat/hiframes/pd_dataframe_ext.py +++ b/hpat/hiframes/pd_dataframe_ext.py @@ -170,7 +170,7 @@ def resolve_apply(self, df, args, kws): row_typ = types.NamedTuple(dtypes, Row) code = func.literal_value.code f_ir = numba.ir_utils.get_ir_of_code({'np': np}, code) - _, f_return_type, _ = numba.compiler.type_inference_stage( + _, f_return_type, _ = numba.typed_passes.type_inference_stage( self.context, f_ir, (row_typ,), None) return signature(SeriesType(f_return_type), *args) diff --git a/hpat/hiframes/pd_groupby_ext.py b/hpat/hiframes/pd_groupby_ext.py index ad0937da8..eb7de2e27 100644 --- a/hpat/hiframes/pd_groupby_ext.py +++ b/hpat/hiframes/pd_groupby_ext.py @@ -140,7 +140,7 @@ def _get_agg_typ(self, grp, args, code): out_columns.append(c) ind = grp.df_type.columns.index(c) data = grp.df_type.data[ind] - _, out_dtype, _ = numba.compiler.type_inference_stage( + _, out_dtype, _ = numba.typed_passes.type_inference_stage( self.context, f_ir, (data,), None) out_arr = _get_series_array_type(out_dtype) out_data.append(out_arr) @@ -228,7 +228,7 @@ def generic(self, args, kws): func = get_agg_func(None, aggfunc.literal_value, None) f_ir = numba.ir_utils.get_ir_of_code( {'np': np, 'numba': numba, 'hpat': hpat}, func.__code__) - _, out_dtype, _ = numba.compiler.type_inference_stage( + _, out_dtype, _ = numba.typed_passes.type_inference_stage( self.context, f_ir, (data,), None) out_arr_typ = _get_series_array_type(out_dtype) diff --git a/hpat/hiframes/pd_series_ext.py b/hpat/hiframes/pd_series_ext.py index d36986836..a8e59c462 100644 --- a/hpat/hiframes/pd_series_ext.py +++ b/hpat/hiframes/pd_series_ext.py @@ -571,7 +571,7 @@ def _resolve_map_func(self, ary, args, kws): _globals = args[0].literal_value.globals f_ir = numba.ir_utils.get_ir_of_code(_globals, code) - f_typemap, f_return_type, f_calltypes = numba.compiler.type_inference_stage( + f_typemap, f_return_type, f_calltypes = numba.typed_passes.type_inference_stage( self.context, f_ir, (dtype,), None) return signature(SeriesType(f_return_type), *args) @@ -595,7 +595,7 @@ def _resolve_combine_func(self, ary, args, kws): dtype2 = pandas_timestamp_type code = args[1].literal_value.code f_ir = numba.ir_utils.get_ir_of_code({'np': np}, code) - f_typemap, f_return_type, f_calltypes = numba.compiler.type_inference_stage( + f_typemap, f_return_type, f_calltypes = numba.typed_passes.type_inference_stage( self.context, f_ir, (dtype1, dtype2,), None) return signature(SeriesType(f_return_type), *args) diff --git a/hpat/hiframes/sort.py b/hpat/hiframes/sort.py index dbc0c9bab..262e5e03b 100644 --- a/hpat/hiframes/sort.py +++ b/hpat/hiframes/sort.py @@ -315,7 +315,7 @@ def sort_distributed_run(sort_node, array_dists, typemap, calltypes, typingctx, func_text += " return key_arrs, data\n" loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) sort_impl = loc_vars['f'] key_typ = types.Tuple([typemap[v.name] for v in key_arrs]) diff --git a/hpat/io/parquet_pio.py b/hpat/io/parquet_pio.py index 1ae5ee457..1d45b784b 100644 --- a/hpat/io/parquet_pio.py +++ b/hpat/io/parquet_pio.py @@ -173,7 +173,7 @@ def get_column_read_nodes(c_type, cvar, arrow_readers_var, i): i, _type_to_pq_dtype_number[el_type]) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat, 'np': np}, loc_vars) size_func = loc_vars['f'] _, f_block = compile_to_numba_ir(size_func, {'get_column_size_parquet': get_column_size_parquet, diff --git a/hpat/io/pio.py b/hpat/io/pio.py index 33f883e67..9886cd647 100644 --- a/hpat/io/pio.py +++ b/hpat/io/pio.py @@ -46,7 +46,7 @@ def handle_possible_h5_read(self, assign, lhs, rhs): # TODO: index arg? func_text += " arr = hpat.io.pio_api.h5_read_dummy(dset, {}, '{}', index)\n".format(tp.ndim, dtype_str) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'hpat': hpat}, loc_vars) _h5_read_impl = loc_vars['_h5_read_impl'] f_block = compile_to_numba_ir(_h5_read_impl, {'hpat': hpat}).blocks.popitem()[1] index_var = rhs.index if rhs.op == 'getitem' else rhs.index_var diff --git a/hpat/io/xenon_ext.py b/hpat/io/xenon_ext.py index 1aeb94c63..46811df0a 100644 --- a/hpat/io/xenon_ext.py +++ b/hpat/io/xenon_ext.py @@ -180,7 +180,7 @@ def get_column_read_nodes(c_type, cvar, xe_connect_var, xe_dset_var, i, schema_a func_text += ' column = np.empty(col_size, dtype=np.{})\n'.format(el_type) func_text += ' status = read_xenon_col(xe_connect_var, xe_dset_var, {}, column, schema_arr)\n'.format(i) loc_vars = {} - exec(func_text, {}, loc_vars) + exec(func_text, {'np': np}, loc_vars) size_func = loc_vars['f'] _, f_block = compile_to_numba_ir(size_func, {'get_column_size_xenon': get_column_size_xenon, diff --git a/hpat/tests/test_basic.py b/hpat/tests/test_basic.py index a1ed7177c..a12ee076a 100644 --- a/hpat/tests/test_basic.py +++ b/hpat/tests/test_basic.py @@ -7,7 +7,7 @@ import random from hpat.tests.test_utils import (count_array_REPs, count_parfor_REPs, count_parfor_OneDs, count_array_OneDs, count_array_OneD_Vars, - dist_IR_contains, get_rank, get_start_end) + dist_IR_contains, get_rank, get_start_end, check_numba_version) def get_np_state_ptr(): @@ -297,6 +297,7 @@ def test_array_reduce(self): self.assertEqual(count_array_OneDs(), 0) self.assertEqual(count_parfor_OneDs(), 1) + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_dist_return(self): def test_impl(N): A = np.arange(N) @@ -313,6 +314,7 @@ def test_impl(N): self.assertEqual(count_array_OneDs(), 1) self.assertEqual(count_parfor_OneDs(), 1) + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_dist_return_tuple(self): def test_impl(N): A = np.arange(N) @@ -341,6 +343,7 @@ def test_impl(A): np.testing.assert_allclose(hpat_func(arr) / self.num_ranks, test_impl(arr)) self.assertEqual(count_array_OneDs(), 1) + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_rebalance(self): def test_impl(N): A = np.arange(n) @@ -358,6 +361,7 @@ def test_impl(N): finally: hpat.distributed_analysis.auto_rebalance = False + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_rebalance_loop(self): def test_impl(N): A = np.arange(n) diff --git a/hpat/tests/test_dataframe.py b/hpat/tests/test_dataframe.py index 47c0f5e18..8904dfa72 100644 --- a/hpat/tests/test_dataframe.py +++ b/hpat/tests/test_dataframe.py @@ -9,7 +9,7 @@ import numba import hpat from hpat.tests.test_utils import (count_array_REPs, count_parfor_REPs, count_parfor_OneDs, - count_array_OneDs, dist_IR_contains, get_start_end) + count_array_OneDs, dist_IR_contains, get_start_end, check_numba_version) from hpat.tests.gen_test_data import ParquetGenerator from numba.config import IS_32BITS @@ -133,6 +133,7 @@ def test_impl(df): dtype=pd.api.types.CategoricalDtype(['N', 'Y']))}) pd.testing.assert_frame_equal(hpat_func(df.copy(deep=True)), test_impl(df)) + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_box_dist_return(self): def test_impl(n): df = pd.DataFrame({'A': np.ones(n), 'B': np.arange(n)}) diff --git a/hpat/tests/test_ml.py b/hpat/tests/test_ml.py index d974dc89a..356ad66de 100644 --- a/hpat/tests/test_ml.py +++ b/hpat/tests/test_ml.py @@ -7,7 +7,7 @@ from hpat.tests.test_utils import (count_array_REPs, count_parfor_REPs, count_parfor_OneDs, count_array_OneDs, count_parfor_OneD_Vars, count_array_OneD_Vars, - dist_IR_contains) + dist_IR_contains, check_numba_version) class TestML(unittest.TestCase): @@ -90,6 +90,7 @@ def test_impl(n): self.assertEqual(count_array_OneDs(), 1) self.assertEqual(count_parfor_OneDs(), 2) + @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690") def test_kmeans(self): def test_impl(numCenter, numIter, N, D): A = np.ones((N, D)) diff --git a/hpat/tests/test_series.py b/hpat/tests/test_series.py index 2a27408ea..5d3491077 100644 --- a/hpat/tests/test_series.py +++ b/hpat/tests/test_series.py @@ -4,7 +4,6 @@ import numpy as np import pyarrow.parquet as pq import hpat - from hpat.tests.test_utils import ( count_array_REPs, count_parfor_REPs, count_array_OneDs, get_start_end) from hpat.tests.gen_test_data import ParquetGenerator diff --git a/hpat/tests/test_utils.py b/hpat/tests/test_utils.py index 33ab17075..4a60dd4bf 100644 --- a/hpat/tests/test_utils.py +++ b/hpat/tests/test_utils.py @@ -1,5 +1,5 @@ import hpat - +import numba def count_array_REPs(): from hpat.distributed import Distribution @@ -53,3 +53,6 @@ def get_start_end(n): start = hpat.distributed_api.get_start(n, n_pes, rank) end = hpat.distributed_api.get_end(n, n_pes, rank) return start, end + +def check_numba_version(version): + return numba.__version__ == version diff --git a/hpat/utils.py b/hpat/utils.py index 50ac33e44..e99093999 100644 --- a/hpat/utils.py +++ b/hpat/utils.py @@ -18,6 +18,7 @@ from hpat.str_ext import string_type, list_string_array_type from hpat.str_arr_ext import string_array_type, num_total_chars, pre_alloc_string_array from enum import Enum +import types as pytypes # int values for types to pass to C code @@ -531,3 +532,7 @@ def dump_node_list(node_list): def debug_prints(): return numba.config.DEBUG_ARRAY_OPT == 1 + +def update_globals(func, glbls): + if isinstance(func, pytypes.FunctionType): + func.__globals__.update(glbls)