Merge from gold/2021 to dev branch (excluding recipe changes)

README.rst

@@ -85,7 +85,7 @@ Building on Linux with setuptools
 
     export PYVER=<3.6 or 3.7>
     export NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.53.1 pandas=1.2.0 pyarrow=4.0.1 gcc_linux-64 gxx_linux-64
+    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER tbb-devel tbb4py numba=0.54.1 pandas=1.2.0 pyarrow=4.0.1 gcc_linux-64 gxx_linux-64
     source activate sdc-env
     git clone https://github.com/IntelPython/sdc.git
     cd sdc
@@ -123,7 +123,7 @@ Building on Windows with setuptools
 
     set PYVER=<3.6 or 3.7>
     set NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.53.1 pandas=1.2.0 pyarrow=4.0.1
+    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% tbb-devel tbb4py numba=0.54.1 pandas=1.2.0 pyarrow=4.0.1
     conda activate sdc-env
     set INCLUDE=%INCLUDE%;%CONDA_PREFIX%\Library\include
     set LIB=%LIB%;%CONDA_PREFIX%\Library\lib

azure-pipelines.yml

@@ -31,8 +31,8 @@ jobs:
 
 - template: buildscripts/azure/template-linux-macos.yml
   parameters:
-    name: Ubuntu1604
-    vmImage: ubuntu-16.04
+    name: Ubuntu1804
+    vmImage: ubuntu-18.04
     allowFailure: false
     matrix:
       py3.7_numpy1.17:

conda-recipe/meta.yaml

@@ -1,4 +1,4 @@
-{% set NUMBA_VERSION = "==0.53.1" %}
+{% set NUMBA_VERSION = "==0.54.1" %}
 {% set PANDAS_VERSION = "==1.2.0" %}
 {% set PYARROW_VERSION = "==4.0.1" %}
 
@@ -33,7 +33,7 @@ requirements:
 
   run:
     - python
-    - numpy
+    - {{ pin_compatible('numpy') }}
     - pandas  {{ PANDAS_VERSION }}
     - pyarrow {{ PYARROW_VERSION }}
     - numba   {{ NUMBA_VERSION }}

requirements.txt

@@ -1,6 +1,6 @@
 numpy>=1.16
 pandas==1.2.0
 pyarrow==4.0.1
-numba==0.53.1
+numba==0.54.1
 tbb
 tbb-devel

sdc/_str_ext.cpp

@@ -32,6 +32,8 @@
 #include <vector>
 #include <cmath>
 #include <algorithm>
+#include <stdexcept>
+#include <limits>
 
 #include "_str_decode.cpp"
 

sdc/cv_ext.py

@@ -75,7 +75,7 @@ def lower_cv2_imread(context, builder, sig, args):
                             [ll_shty.as_pointer(),
                              lir.IntType(8).as_pointer().as_pointer(),
                              lir.IntType(8).as_pointer()])
-    fn_imread = builder.module.get_or_insert_function(fnty, name="cv_imread")
+    fn_imread = cgutils.get_or_insert_function(builder.module, fnty, name="cv_imread")
     img = builder.call(fn_imread, [shapes_array, data, fname])
 
     return _image_to_array(context, builder, shapes_array, arrtype, data, img)
@@ -99,7 +99,7 @@ def lower_cv2_imread(context, builder, sig, args):
 #                              lir.IntType(8).as_pointer(),
 #                              lir.IntType(64),
 #                              lir.IntType(64)])
-#     fn_resize = builder.module.get_or_insert_function(fnty, name="cv_resize")
+#     fn_resize = cgutils.get_or_insert_function(builder.module, fnty, name="cv_resize")
 #     img = builder.call(fn_resize, [new_sizes[1], new_sizes[0], ary.data, in_array.data,
 #                                     in_shapes[0], in_shapes[1]])
 #
@@ -115,7 +115,7 @@ def _image_to_array(context, builder, shapes_array, arrtype, data, img):
 
     # clean up cv::Mat image
     fnty = lir.FunctionType(lir.VoidType(), [lir.IntType(8).as_pointer()])
-    fn_release = builder.module.get_or_insert_function(fnty, name="cv_mat_release")
+    fn_release = cgutils.get_or_insert_function(builder.module, fnty, name="cv_mat_release")
     builder.call(fn_release, [img])
 
     return impl_ret_new_ref(context, builder, arrtype, ary._getvalue())
@@ -203,7 +203,7 @@ def codegen(context, builder, sig, args):
 
         # clean up image buffer
         fnty = lir.FunctionType(lir.VoidType(), [lir.IntType(8).as_pointer()])
-        fn_release = builder.module.get_or_insert_function(fnty, name="cv_delete_buf")
+        fn_release = cgutils.get_or_insert_function(builder.module, fnty, name="cv_delete_buf")
         builder.call(fn_release, [data])
 
         return impl_ret_new_ref(context, builder, sig.return_type, ary._getvalue())

sdc/datatypes/common_functions.py

@@ -455,35 +455,29 @@ def sdc_join_series_indexes_impl(left, right):
     return None
 
 
-@numba.njit
 def _sdc_pandas_format_percentiles(arr):
+    pass
+
+
+@sdc_overload(_sdc_pandas_format_percentiles)
+def _sdc_pandas_format_percentiles_ovld(arr):
     """ Function converting float array of percentiles to a list of strings formatted
         the same as in pandas.io.formats.format.format_percentiles
     """
 
-    percentiles_strs = []
-    for percentile in arr:
-        p_as_string = str(percentile * 100)
+    dtype_precision = numpy.finfo(arr.dtype.key).precision
 
-        trim_index = len(p_as_string) - 1
-        while trim_index >= 0:
-            if p_as_string[trim_index] == '0':
-                trim_index -= 1
-                continue
-            elif p_as_string[trim_index] == '.':
-                break
-
-            trim_index += 1
-            break
-
-        if trim_index < 0:
-            p_as_string_trimmed = '0'
-        else:
-            p_as_string_trimmed = p_as_string[:trim_index]
+    def _sdc_pandas_format_percentiles_impl(arr):
+        percentiles_strs = []
+        for percentile in arr:
+            p_as_string = str(numpy.round(percentile * 100, dtype_precision - 1))
+            p_as_string_trimmed = p_as_string.rstrip('0')
+            p_as_string_trimmed = p_as_string_trimmed.rstrip('.')
+            percentiles_strs.append(p_as_string_trimmed + '%')
 
-        percentiles_strs.append(p_as_string_trimmed + '%')
+        return percentiles_strs
 
-    return percentiles_strs
+    return _sdc_pandas_format_percentiles_impl
 
 
 def sdc_arrays_argsort(A, kind='quicksort'):

sdc/datatypes/hpat_pandas_series_functions.py

@@ -4429,7 +4429,7 @@ def hpat_pandas_series_describe_numeric_impl(self, percentiles=None, include=Non
                 sorted_percentiles = sorted(percentiles_list)
 
                 # check percentiles have correct values:
-                arr = numpy.asarray(sorted_percentiles)
+                arr = numpy.asarray(sorted_percentiles).astype(numpy.float64)
                 if len(numpy.unique(arr)) != len(arr):
                     raise ValueError("percentiles cannot contain duplicates")
                 if numpy.any(arr[(arr < 0) * (arr > 1)]):

sdc/distributed_lower.py

@@ -89,14 +89,14 @@
 @lower_builtin(distributed_api.get_rank)
 def dist_get_rank(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(32), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_get_rank")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_get_rank")
     return builder.call(fn, [])
 
 
 @lower_builtin(distributed_api.get_size)
 def dist_get_size(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(32), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_get_size")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_get_size")
     return builder.call(fn, [])
 
 
@@ -113,7 +113,7 @@ def dist_get_start(context, builder, sig, args):
 def dist_get_end(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(64), [lir.IntType(64),
                                               lir.IntType(32), lir.IntType(32)])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_get_end")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_get_end")
     return builder.call(fn, [args[0], args[1], args[2]])
 
 
@@ -163,7 +163,7 @@ def lower_dist_reduce(context, builder, sig, args):
 
     fnty = lir.FunctionType(lir.VoidType(), [lir.IntType(8).as_pointer(),
                                              lir.IntType(8).as_pointer(), op_typ, lir.IntType(32)])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_reduce")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_reduce")
     builder.call(fn, [in_ptr, out_ptr, args[1], builder.load(typ_arg)])
     # cast back to value type
     out_ptr = builder.bitcast(out_ptr, val_typ.as_pointer())
@@ -206,21 +206,21 @@ def lower_dist_arr_reduce(context, builder, sig, args):
 @lower_builtin(time.time)
 def dist_get_time(context, builder, sig, args):
     fnty = lir.FunctionType(lir.DoubleType(), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_get_time")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_get_time")
     return builder.call(fn, [])
 
 
 @lower_builtin(distributed_api.dist_time)
 def dist_get_dist_time(context, builder, sig, args):
     fnty = lir.FunctionType(lir.DoubleType(), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_get_time")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_get_time")
     return builder.call(fn, [])
 
 
 @lower_builtin(distributed_api.barrier)
 def dist_barrier(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(32), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_barrier")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_barrier")
     return builder.call(fn, [])
 
 
@@ -285,7 +285,7 @@ def lower_dist_irecv(context, builder, sig, args):
                     32), lir.IntType(32), lir.IntType(32),
                 lir.IntType(1)]
     fnty = lir.FunctionType(mpi_req_llvm_type, arg_typs)
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_irecv")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_irecv")
     return builder.call(fn, call_args)
 
 # array, size, pe, tag, cond
@@ -313,22 +313,22 @@ def lower_dist_isend(context, builder, sig, args):
                     32), lir.IntType(32), lir.IntType(32),
                 lir.IntType(1)]
     fnty = lir.FunctionType(mpi_req_llvm_type, arg_typs)
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_isend")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_isend")
     return builder.call(fn, call_args)
 
 
 @lower_builtin(distributed_api.wait, mpi_req_numba_type, types.boolean)
 def lower_dist_wait(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(32), [mpi_req_llvm_type, lir.IntType(1)])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_wait")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_wait")
     return builder.call(fn, args)
 
 
 @lower_builtin(distributed_api.waitall, types.int32, req_array_type)
 def lower_dist_waitall(context, builder, sig, args):
     fnty = lir.FunctionType(lir.VoidType(),
                             [lir.IntType(32), lir.IntType(8).as_pointer()])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_waitall")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_waitall")
     builder.call(fn, args)
     return context.get_dummy_value()
 
@@ -425,22 +425,22 @@ def lower_dist_allgather(context, builder, sig, args):
 
     fnty = lir.FunctionType(lir.VoidType(), [lir.IntType(8).as_pointer(),
                                              lir.IntType(32), val_ptr.type, lir.IntType(32)])
-    fn = builder.module.get_or_insert_function(fnty, name="allgather")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="allgather")
     builder.call(fn, call_args)
     return context.get_dummy_value()
 
 
 @lower_builtin(distributed_api.comm_req_alloc, types.int32)
 def lower_dist_comm_req_alloc(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(8).as_pointer(), [lir.IntType(32)])
-    fn = builder.module.get_or_insert_function(fnty, name="comm_req_alloc")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="comm_req_alloc")
     return builder.call(fn, args)
 
 
 @lower_builtin(distributed_api.comm_req_dealloc, req_array_type)
 def lower_dist_comm_req_dealloc(context, builder, sig, args):
     fnty = lir.FunctionType(lir.VoidType(), [lir.IntType(8).as_pointer()])
-    fn = builder.module.get_or_insert_function(fnty, name="comm_req_dealloc")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="comm_req_dealloc")
     builder.call(fn, args)
     return context.get_dummy_value()
 
@@ -471,7 +471,7 @@ def setitem_req_array(context, builder, sig, args):
 #                       wraparound=False):
 #         # get local index or -1 if out of bounds
 #         fnty = lir.FunctionType(lir.IntType(64), [lir.IntType(64), lir.IntType(64), lir.IntType(64)])
-#         fn = builder.module.get_or_insert_function(fnty, name="hpat_dist_get_item_pointer")
+#         fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_dist_get_item_pointer")
 #         first_ind = builder.call(fn, [inds[0], start, count])
 #         inds = tuple([first_ind, *inds[1:]])
 #         # regular local pointer with new indices
@@ -481,7 +481,7 @@ def setitem_req_array(context, builder, sig, args):
 #         not_inbound = builder.icmp_signed('==', first_ind, lir.Constant(lir.IntType(64), -1))
 #         # get dummy pointer
 #         dummy_fnty  = lir.FunctionType(lir.IntType(8).as_pointer(), [])
-#         dummy_fn = builder.module.get_or_insert_function(dummy_fnty, name="hpat_get_dummy_ptr")
+#         dummy_fn = cgutils.get_or_insert_function(builder.module, dummy_fnty, name="hpat_get_dummy_ptr")
 #         dummy_ptr = builder.bitcast(builder.call(dummy_fn, []), in_ptr.type)
 #         with builder.if_then(not_inbound, likely=True):
 #             builder.store(dummy_ptr, ret_ptr)
@@ -613,7 +613,7 @@ def generic(self, args, kws):
 @lower_builtin(hpat_finalize)
 def lower_hpat_finalize(context, builder, sig, args):
     fnty = lir.FunctionType(lir.IntType(32), [])
-    fn = builder.module.get_or_insert_function(fnty, name="hpat_finalize")
+    fn = cgutils.get_or_insert_function(builder.module, fnty, name="hpat_finalize")
     return builder.call(fn, args)
 
 

sdc/extensions/indexes/int64_index_ext.py

@@ -40,7 +40,7 @@
 
 from sdc.datatypes.indexes import *
 from sdc.utilities.sdc_typing_utils import SDCLimitation
-from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method, BooleanLiteral
+from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method
 from sdc.utilities.sdc_typing_utils import (
         TypeChecker,
         check_signed_integer,
@@ -388,10 +388,12 @@ def pd_int64_index_is_overload(context, builder, sig, args):
     if ty_lhs != ty_rhs:
         return cgutils.false_bit
 
+    # llvmlite passes LiteralStructs into functions as separate fields
+    # and there's no way to get pointer to original struct (where it's allocated)
+    # other than walk through chain of instruction operands to alloca instruction
+    # so just check if the instructions themselves match exactly
     lhs, rhs = args
-    lhs_ptr = builder.ptrtoint(lhs.operands[0], cgutils.intp_t)
-    rhs_ptr = builder.ptrtoint(rhs.operands[0], cgutils.intp_t)
-    return builder.icmp_signed('==', lhs_ptr, rhs_ptr)
+    return context.get_constant(types.bool_, lhs == rhs)
 
 
 @lower_builtin('getiter', Int64IndexType)
@@ -543,7 +545,7 @@ def pd_int64_index_join_overload(self, other, how, level=None, return_indexers=F
     if not (isinstance(level, (types.Omitted, types.NoneType)) or level is None):
         ty_checker.raise_exc(level, 'None', 'level')
 
-    if not (isinstance(return_indexers, (types.Omitted, BooleanLiteral)) or return_indexers is False):
+    if not (isinstance(return_indexers, (types.Omitted, types.BooleanLiteral)) or return_indexers is False):
         ty_checker.raise_exc(return_indexers, 'boolean', 'return_indexers')
 
     if not (isinstance(sort, (types.Omitted, types.Boolean)) or sort is False):

sdc/extensions/indexes/multi_index_ext.py

@@ -42,7 +42,7 @@
 
 from sdc.datatypes.indexes import *
 from sdc.utilities.sdc_typing_utils import SDCLimitation
-from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method, BooleanLiteral
+from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method
 from sdc.utilities.sdc_typing_utils import (
         TypeChecker,
         sdc_pandas_index_types,
@@ -759,10 +759,9 @@ def pd_multi_index_is_overload(context, builder, sig, args):
     if ty_lhs != ty_rhs:
         return cgutils.false_bit
 
+    # similar to Int64Index (compare instructions building index structs)
     lhs, rhs = args
-    lhs_ptr = builder.ptrtoint(lhs.operands[0], cgutils.intp_t)
-    rhs_ptr = builder.ptrtoint(rhs.operands[0], cgutils.intp_t)
-    return builder.icmp_signed('==', lhs_ptr, rhs_ptr)
+    return context.get_constant(types.bool_, lhs == rhs)
 
 
 @lower_builtin('getiter', MultiIndexType)

sdc/extensions/indexes/positional_index_ext.py

@@ -40,7 +40,7 @@
 from sdc.datatypes.indexes import PositionalIndexType, RangeIndexType
 from sdc.datatypes.indexes.range_index_type import RangeIndexDataType
 from sdc.utilities.sdc_typing_utils import SDCLimitation
-from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method, BooleanLiteral
+from sdc.utilities.utils import sdc_overload, sdc_overload_attribute, sdc_overload_method
 from sdc.extensions.indexes.range_index_ext import box_range_index, unbox_range_index
 from sdc.utilities.sdc_typing_utils import (
                                 TypeChecker,
@@ -300,10 +300,9 @@ def pd_positional_index_is_overload(context, builder, sig, args):
     if ty_lhs != ty_rhs:
         return cgutils.false_bit
 
+    # similar to Int64Index (compare instructions building index structs)
     lhs, rhs = args
-    lhs_ptr = builder.ptrtoint(lhs.operands[0], cgutils.intp_t)
-    rhs_ptr = builder.ptrtoint(rhs.operands[0], cgutils.intp_t)
-    return builder.icmp_signed('==', lhs_ptr, rhs_ptr)
+    return context.get_constant(types.bool_, lhs == rhs)
 
 
 @lower_builtin('getiter', PositionalIndexType)
@@ -433,7 +432,7 @@ def pd_positional_index_join_overload(self, other, how, level=None, return_index
     if not (isinstance(level, (types.Omitted, types.NoneType)) or level is None):
         ty_checker.raise_exc(level, 'None', 'level')
 
-    if not (isinstance(return_indexers, (types.Omitted, BooleanLiteral)) or return_indexers is False):
+    if not (isinstance(return_indexers, (types.Omitted, types.BooleanLiteral)) or return_indexers is False):
         ty_checker.raise_exc(return_indexers, 'boolean', 'return_indexers')
 
     if not (isinstance(sort, (types.Omitted, types.Boolean)) or sort is False):