Merge 3d64d4f into 7dbac00

hpat/compiler.py

@@ -27,6 +27,7 @@
 # workaround for Numba #3876 issue with large labels in mortgage benchmark
 binding.set_option("tmp", "-non-global-value-max-name-size=2048")
 
+
 def inline_calls(func_ir, _locals):
     work_list = list(func_ir.blocks.items())
     while work_list:
@@ -63,7 +64,7 @@ def inline_calls(func_ir, _locals):
     # CFG simplification fixes this case
     func_ir.blocks = ir_utils.simplify_CFG(func_ir.blocks)
 
-#TODO: remove these helper functions when Numba provide appropriate way to manipulate passes
+# TODO: remove these helper functions when Numba provide appropriate way to manipulate passes
 def pass_position(pm, location):
     assert pm.passes
     pm._validate_pass(location)
@@ -83,6 +84,7 @@ def add_pass_before(pm, pass_cls, location):
     # if a pass has been added, it's not finalized
     pm._finalized = False
 
+
 def replace_pass(pm, pass_cls, location):
     assert pm.passes
     pm._validate_pass(pass_cls)
@@ -92,6 +94,7 @@ def replace_pass(pm, pass_cls, location):
     # if a pass has been added, it's not finalized
     pm._finalized = False
 
+
 @register_pass(mutates_CFG=True, analysis_only=False)
 class InlinePass(FunctionPass):
     _name = "hpat_inline_pass"
@@ -103,6 +106,7 @@ def run_pass(self, state):
         inline_calls(state.func_ir, state.locals)
         return True
 
+
 @register_pass(mutates_CFG=True, analysis_only=False)
 class PostprocessorPass(FunctionPass):
     _name = "hpat_postprocessor_pass"
@@ -115,6 +119,7 @@ def run_pass(self, state):
         post_proc.run()
         return True
 
+
 class HPATPipeline(numba.compiler.CompilerBase):
     """HPAT compiler pipeline
     """
@@ -147,13 +152,14 @@ def run_pass(self, state):
 
         return True
 
+
 class HPATPipelineSeq(HPATPipeline):
     """HPAT pipeline without the distributed pass (used in rolling kernels)
     """
 
     def define_pipelines(self):
         name = 'hpat_seq'
-        pm   = DefaultPassBuilder.define_nopython_pipeline(self.state)
+        pm = DefaultPassBuilder.define_nopython_pipeline(self.state)
 
         add_pass_before(pm, InlinePass, InlineClosureLikes)
         pm.add_pass_after(HiFramesPass, InlinePass)
@@ -163,4 +169,4 @@ def define_pipelines(self):
         add_pass_before(pm, ParforSeqPass, IRLegalization)
         pm.finalize()
 
-        return [pm]
+        return [pm]

hpat/datatypes/hpat_pandas_series_functions.py

@@ -463,7 +463,6 @@ def hpat_pandas_series_T_impl(self):
     return hpat_pandas_series_T_impl
 
 
-
 @overload(len)
 def hpat_pandas_series_len(self):
     """
@@ -663,9 +662,7 @@ def hpat_pandas_series_copy(self, deep=True):
     """
     _func_name = 'Method Series.copy().'
 
-    if (isinstance(self, SeriesType) and
-        (isinstance(deep, (types.Omitted, types.Boolean)) or deep == True)
-    ):
+    if (isinstance(self, SeriesType) and (isinstance(deep, (types.Omitted, types.Boolean)) or deep)):
         if isinstance(self.index, types.NoneType):
             def hpat_pandas_series_copy_impl(self, deep=True):
                 if deep:
@@ -1110,20 +1107,26 @@ def hpat_pandas_series_sum(
     if not isinstance(self, SeriesType):
         raise TypingError('{} The object must be a pandas.series. Given: {}'.format(_func_name, self))
 
-    if not (isinstance(axis, (types.Integer, types.Omitted)) or axis == None):
+    if not (isinstance(axis, (types.Integer, types.Omitted)) or axis is None):
         raise TypingError('{} The axis must be an Integer. Currently unsupported. Given: {}'.format(_func_name, axis))
 
-    if not (isinstance(skipna, (types.Boolean, types.Omitted)) or skipna == None):
+    if not (isinstance(skipna, (types.Boolean, types.Omitted)) or skipna is None):
         raise TypingError('{} The skipna must be a Boolean. Given: {}'.format(_func_name, skipna))
 
-    if not (isinstance(level, (types.Integer, types.StringLiteral, types.Omitted)) or level == None):
-        raise TypingError('{} The level must be an Integer or level name. Currently unsupported. Given: {}'.format(_func_name, level))
+    if not (isinstance(level, (types.Integer, types.StringLiteral, types.Omitted)) or level is None):
+        raise TypingError(
+            '{} The level must be an Integer or level name. Currently unsupported. Given: {}'.format(
+                _func_name, level))
 
-    if not (isinstance(numeric_only, (types.Boolean, types.Omitted)) or numeric_only == None):
-        raise TypingError('{} The numeric_only must be a Boolean. Currently unsupported. Given: {}'.format(_func_name, numeric_only))
+    if not (isinstance(numeric_only, (types.Boolean, types.Omitted)) or numeric_only is None):
+        raise TypingError(
+            '{} The numeric_only must be a Boolean. Currently unsupported. Given: {}'.format(
+                _func_name, numeric_only))
 
     if not (isinstance(min_count, (types.Integer, types.Omitted)) or min_count == 0):
-        raise TypingError('{} The min_count must be an Integer. Currently unsupported. Given: {}'.format(_func_name, min_count))
+        raise TypingError(
+            '{} The min_count must be an Integer. Currently unsupported. Given: {}'.format(
+                _func_name, min_count))
 
     def hpat_pandas_series_sum_impl(
         self,
@@ -1634,8 +1637,9 @@ def hpat_pandas_series_min(self, axis=None, skipna=True, level=None, numeric_onl
         raise TypingError('{} The object must be a pandas.series. Given: {}'.format(_func_name, self))
 
     if not isinstance(self.data.dtype, (types.Integer, types.Float)):
-        raise TypingError('{} Currently function supports only numeric values. Given data type: {}'.format(_func_name,
-                                                                                                           self.data.dtype))
+        raise TypingError(
+            '{} Currently function supports only numeric values. Given data type: {}'.format(
+                _func_name, self.data.dtype))
 
     if not isinstance(skipna, (types.Omitted, types.Boolean)) and skipna is not True:
         raise TypingError(
@@ -1686,8 +1690,9 @@ def hpat_pandas_series_max(self, axis=None, skipna=True, level=None, numeric_onl
         raise TypingError('{} The object must be a pandas.series. Given: {}'.format(_func_name, self))
 
     if not isinstance(self.data.dtype, (types.Integer, types.Float)):
-        raise TypingError('{} Currently function supports only numeric values. Given data type: {}'.format(_func_name,
-                                                                                                           self.data.dtype))
+        raise TypingError(
+            '{} Currently function supports only numeric values. Given data type: {}'.format(
+                _func_name, self.data.dtype))
 
     if not isinstance(skipna, (types.Omitted, types.Boolean)) and skipna is not True:
         raise TypingError(
@@ -1745,7 +1750,9 @@ def hpat_pandas_series_mean(self, axis=None, skipna=None, level=None, numeric_on
         raise TypingError('{} The object must be a pandas.series. Given: {}'.format(_func_name, self))
 
     if not isinstance(self.data.dtype, types.Number):
-        raise TypingError('{} Currently function supports only numeric values. Given data type: {}'.format(_func_name, self.data.dtype))
+        raise TypingError(
+            '{} Currently function supports only numeric values. Given data type: {}'.format(
+                _func_name, self.data.dtype))
 
     if not isinstance(skipna, (types.Omitted, types.Boolean)) and skipna is not None:
         raise TypingError(
@@ -2390,7 +2397,6 @@ def hpat_pandas_series_nunique_impl(self, dropna=True):
 
 @overload_method(SeriesType, 'count')
 def hpat_pandas_series_count(self, level=None):
-
     """
     Pandas Series method :meth:`pandas.Series.count` implementation.
     .. only:: developer
@@ -2477,20 +2483,24 @@ def hpat_pandas_series_median(self, axis=None, skipna=True, level=None, numeric_
 
     if not isinstance(self.dtype, types.Number):
         raise TypingError(
-            '{} The function only applies to elements that are all numeric. Given data type: {}'.format(_func_name, self.dtype))
+            '{} The function only applies to elements that are all numeric. Given data type: {}'.format(
+                _func_name, self.dtype))
 
     if not (isinstance(axis, (types.Integer, types.UnicodeType, types.Omitted)) or axis is None):
-        raise TypingError('{} The axis must be an Integer or a String. Currently unsupported. Given: {}'.format(_func_name, axis))
+        raise TypingError(
+            '{} The axis must be an Integer or a String. Currently unsupported. Given: {}'.format(
+                _func_name, axis))
 
-    if not (isinstance(skipna, (types.Boolean, types.Omitted)) or skipna == True):
+    if not (isinstance(skipna, (types.Boolean, types.Omitted)) or skipna):
         raise TypingError('{} The is_copy must be a boolean. Given: {}'.format(_func_name, skipna))
 
     if not ((level is None or isinstance(level, types.Omitted))
             or (numeric_only is None or isinstance(numeric_only, types.Omitted))
             or (axis is None or isinstance(axis, types.Omitted))
-    ):
-        raise TypingError('{} Unsupported parameters. Given level: {}, numeric_only: {}, axis: {}'.format(_func_name, level, numeric_only, axis))
-
+            ):
+        raise TypingError(
+            '{} Unsupported parameters. Given level: {}, numeric_only: {}, axis: {}'.format(
+                _func_name, level, numeric_only, axis))
 
     def hpat_pandas_series_median_impl(self, axis=None, skipna=True, level=None, numeric_only=None):
         if skipna:
@@ -2538,7 +2548,7 @@ def hpat_pandas_series_dropna(self, axis=0, inplace=False):
         raise TypingError('{} Unsupported parameters. Given inplace: {}'.format(_func_name, inplace))
 
     def hpat_pandas_series_dropna_impl(self, axis=0, inplace=False):
-        # generate Series index if needed by using SeriesType.index (i.e. not self._index) 
+        # generate Series index if needed by using SeriesType.index (i.e. not self._index)
         na_data_arr = hpat.hiframes.api.get_nan_mask(self._data)
         data = self._data[~na_data_arr]
         index = self.index[~na_data_arr]

hpat/distributed.py

@@ -79,6 +79,7 @@
 dist_analysis = None
 fir_text = None
 
+
 @register_pass(mutates_CFG=True, analysis_only=False)
 class DistributedPass(FunctionPass):
     """The summary of the class should be here for example below is the summary line for this class
@@ -87,6 +88,7 @@ class DistributedPass(FunctionPass):
     """
 
     _name = "distributed_pass"
+
     def __init__(self):
         pass
 
@@ -1954,7 +1956,8 @@ def f(old_slice, offset):  # pragma: no cover
             slice_type = self.state.typemap[slice_var.name]
             arg_typs = (slice_type, types.intp,)
         _globals = self.state.func_ir.func_id.func.__globals__
-        f_ir = compile_to_numba_ir(f, _globals, self.state.typingctx, arg_typs, self.state.typemap, self.state.calltypes)
+        f_ir = compile_to_numba_ir(f, _globals, self.state.typingctx, arg_typs,
+                                   self.state.typemap, self.state.calltypes)
         _, block = f_ir.blocks.popitem()
         replace_arg_nodes(block, args)
         return block.body[:-2]  # ignore return nodes
@@ -2035,7 +2038,8 @@ def _gen_barrier(self):
         def f():  # pragma: no cover
             return hpat.distributed_api.barrier()
 
-        f_blocks = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, {}, self.state.typemap, self.state.calltypes).blocks
+        f_blocks = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx, {},
+                                       self.state.typemap, self.state.calltypes).blocks
         block = f_blocks[min(f_blocks.keys())]
         return block.body[:-2]  # remove return
 
@@ -2047,8 +2051,12 @@ def _gen_reduce(self, reduce_var, reduce_op, scope, loc):
         def f(val, op):  # pragma: no cover
             hpat.distributed_api.dist_reduce(val, op)
 
-        f_ir = compile_to_numba_ir(f, {'hpat': hpat}, self.state.typingctx,
-                                   (self.state.typemap[reduce_var.name], types.int32), self.state.typemap, self.state.calltypes)
+        f_ir = compile_to_numba_ir(f,
+                                   {'hpat': hpat},
+                                   self.state.typingctx,
+                                   (self.state.typemap[reduce_var.name], types.int32),
+                                   self.state.typemap,
+                                   self.state.calltypes)
         _, block = f_ir.blocks.popitem()
 
         replace_arg_nodes(block, [reduce_var, op_var])
@@ -2120,8 +2128,12 @@ def _gen_init_reduce(self, reduce_var, reduce_op):
         exec(f_text, {'hpat': hpat}, loc_vars)
         f = loc_vars['f']
 
-        f_block = compile_to_numba_ir(f, {'hpat': hpat, 'numba': numba, 'np': np},
-                                      self.state.typingctx, (red_var_typ,), self.state.typemap, self.state.calltypes).blocks.popitem()[1]
+        f_block = compile_to_numba_ir(f,
+                                      {'hpat': hpat, 'numba': numba, 'np': np},
+                                      self.state.typingctx,
+                                      (red_var_typ,),
+                                      self.state.typemap,
+                                      self.state.calltypes).blocks.popitem()[1]
         replace_arg_nodes(f_block, [reduce_var])
         nodes = f_block.body[:-3]
         nodes[-1].target = reduce_var

hpat/hiframes/aggregate.py

@@ -1048,8 +1048,8 @@ def gen_top_level_agg_func(key_names, return_key, red_var_typs, out_typs,
 
 def compile_to_optimized_ir(func, arg_typs, typingctx):
     state = namedtuple('State',
-                            ['typingctx', 'targetctx', 'args', 'func_ir', 'typemap', 'return_type',
-                                'calltypes', 'metadata'])
+                       ['typingctx', 'targetctx', 'args', 'func_ir', 'typemap', 'return_type',
+                        'calltypes', 'metadata'])
 
     # XXX are outside function's globals needed?
     code = func.code if hasattr(func, 'code') else func.__code__

hpat/hiframes/dataframe_pass.py

@@ -1675,8 +1675,8 @@ def _run_call_groupby(self, assign, lhs, rhs, grp_var, func_name):
         for c in grp_typ.selection:
             var = ir.Var(lhs.scope, mk_unique_var(c), lhs.loc)
             self.state.typemap[var.name] = (out_typ.data
-                                      if isinstance(out_typ, SeriesType)
-                                      else out_typ.data[out_typ.columns.index(c)])
+                                            if isinstance(out_typ, SeriesType)
+                                            else out_typ.data[out_typ.columns.index(c)])
             df_col_map[c] = var
 
         agg_func = get_agg_func(self.state.func_ir, func_name, rhs)
@@ -1828,8 +1828,8 @@ def _run_call_rolling(self, assign, lhs, rhs, rolling_var, func_name):
         for c in rolling_typ.selection:
             var = ir.Var(lhs.scope, mk_unique_var(c), lhs.loc)
             self.state.typemap[var.name] = (out_typ.data
-                                      if isinstance(out_typ, SeriesType)
-                                      else out_typ.data[out_typ.columns.index(c)])
+                                            if isinstance(out_typ, SeriesType)
+                                            else out_typ.data[out_typ.columns.index(c)])
             df_col_map[c] = var
 
         if on is not None:

hpat/hiframes/hiframes_typed.py

@@ -80,6 +80,7 @@
     '**': '**',
 }
 
+
 @register_pass(mutates_CFG=True, analysis_only=False)
 class HiFramesTypedPass(FunctionPass):
     """Analyze and transform hiframes calls after typing"""

hpat/tests/test_basic.py

@@ -297,7 +297,8 @@ def test_array_reduce(self):
             self.assertEqual(count_array_OneDs(), 0)
             self.assertEqual(count_parfor_OneDs(), 1)
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_dist_return(self):
         def test_impl(N):
             A = np.arange(N)
@@ -314,7 +315,8 @@ def test_impl(N):
         self.assertEqual(count_array_OneDs(), 1)
         self.assertEqual(count_parfor_OneDs(), 1)
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_dist_return_tuple(self):
         def test_impl(N):
             A = np.arange(N)
@@ -343,7 +345,8 @@ def test_impl(A):
         np.testing.assert_allclose(hpat_func(arr) / self.num_ranks, test_impl(arr))
         self.assertEqual(count_array_OneDs(), 1)
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_rebalance(self):
         def test_impl(N):
             A = np.arange(n)
@@ -361,7 +364,8 @@ def test_impl(N):
         finally:
             hpat.distributed_analysis.auto_rebalance = False
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_rebalance_loop(self):
         def test_impl(N):
             A = np.arange(n)

hpat/tests/test_dataframe.py

@@ -133,7 +133,8 @@ def test_impl(df):
                                           dtype=pd.api.types.CategoricalDtype(['N', 'Y']))})
         pd.testing.assert_frame_equal(hpat_func(df.copy(deep=True)), test_impl(df))
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_box_dist_return(self):
         def test_impl(n):
             df = pd.DataFrame({'A': np.ones(n), 'B': np.arange(n)})

hpat/tests/test_ml.py

@@ -90,7 +90,8 @@ def test_impl(n):
         self.assertEqual(count_array_OneDs(), 1)
         self.assertEqual(count_parfor_OneDs(), 2)
 
-    @unittest.skipIf(check_numba_version('0.46.0'), "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
+    @unittest.skipIf(check_numba_version('0.46.0'),
+                     "Broken in numba 0.46.0. https://github.com/numba/numba/issues/4690")
     def test_kmeans(self):
         def test_impl(numCenter, numIter, N, D):
             A = np.ones((N, D))