Merge pull request #5946 from stuartarchibald/wip/const_dict_list

Implement literal dictionaries and lists.

docs/source/reference/pysupported.rst

@@ -509,6 +509,32 @@ of this limitation.
    List sorting currently uses a quicksort algorithm, which has different
    performance characterics than the algorithm used by Python.
 
+.. _feature-list-initial-value:
+
+Initial Values
+''''''''''''''
+.. warning::
+  This is an experimental feature!
+
+Lists that:
+
+* Are constructed using the square braces syntax
+* Have values of a literal type
+
+will have their initial value stored in the ``.initial_value`` property on the
+type so as to permit inspection of these values at compile time. If required,
+to force value based dispatch the :ref:`literally <developer-literally>`
+function will accept such a list.
+
+Example:
+
+.. literalinclude:: ../../../numba/tests/doc_examples/test_literal_container_usage.py
+   :language: python
+   :caption: from ``test_ex_initial_value_list_compile_time_consts`` of ``numba/tests/doc_examples/test_literal_container_usage.py``
+   :start-after: magictoken.test_ex_initial_value_list_compile_time_consts.begin
+   :end-before: magictoken.test_ex_initial_value_list_compile_time_consts.end
+   :dedent: 12
+   :linenos:
 
 .. _feature-typed-list:
 
@@ -584,6 +610,43 @@ Finally, here's an example of using a nested `List()`:
    :dedent: 12
    :linenos:
 
+.. _feature-literal-list:
+
+Literal List
+''''''''''''
+
+.. warning::
+  This is an experimental feature!
+
+Numba supports the use of literal lists containing any values, for example::
+
+  l = ['a', 1, 2j, np.zeros(5,)]
+
+the predominant use of these lists is for use as a configuration object.
+The lists appear as a ``LiteralList`` type which inherits from ``Literal``, as a
+result the literal values of the list items are available at compile time.
+For example:
+
+.. literalinclude:: ../../../numba/tests/doc_examples/test_literal_container_usage.py
+   :language: python
+   :caption: from ``test_ex_literal_list`` of ``numba/tests/doc_examples/test_literal_container_usage.py``
+   :start-after: magictoken.test_ex_literal_list.begin
+   :end-before: magictoken.test_ex_literal_list.end
+   :dedent: 12
+   :linenos:
+
+Important things to note about these kinds of lists:
+
+#. They are immutable, use of mutating methods e.g. ``.pop()`` will result in
+   compilation failure. Read-only static access and read only methods are
+   supported e.g. ``len()``.
+#. Dynamic access of items is not possible, e.g. ``some_list[x]``, for a
+   value ``x`` which is not a compile time constant. This is because it's
+   impossible statically determine the type of the item being accessed.
+#. Inside the compiler, these lists are actually just tuples with some extra
+   things added to make them look like they are lists.
+#. They cannot be returned to the interpreter from a compiled function.
+
 .. _pysupported-comprehension:
 
 List comprehension
@@ -644,7 +707,7 @@ objects of different types, even if the types are compatible (for example,
 .. _feature-typed-dict:
 
 Typed Dict
-''''''''''
+----------
 
 .. warning::
   ``numba.typed.Dict`` is an experimental feature.  The API may change
@@ -695,7 +758,7 @@ instances and letting the compiler infer the key-value types:
 
 .. literalinclude:: ../../../numba/tests/doc_examples/test_typed_dict_usage.py
    :language: python
-   :caption: from ``ex_inferred_dict_njit`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
+   :caption: from ``test_ex_inferred_dict_njit`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
    :start-after: magictoken.ex_inferred_dict_njit.begin
    :end-before: magictoken.ex_inferred_dict_njit.end
    :dedent: 12
@@ -706,7 +769,7 @@ code and using the dictionary in jit code:
 
 .. literalinclude:: ../../../numba/tests/doc_examples/test_typed_dict_usage.py
    :language: python
-   :caption: from ``ex_typed_dict_from_cpython`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
+   :caption: from ``test_ex_typed_dict_from_cpython`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
    :start-after: magictoken.ex_typed_dict_from_cpython.begin
    :end-before: magictoken.ex_typed_dict_from_cpython.end
    :dedent: 12
@@ -717,7 +780,7 @@ using the dictionary in interpreted code:
 
 .. literalinclude:: ../../../numba/tests/doc_examples/test_typed_dict_usage.py
    :language: python
-   :caption: from ``ex_typed_dict_njit`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
+   :caption: from ``test_ex_typed_dict_njit`` of ``numba/tests/doc_examples/test_typed_dict_usage.py``
    :start-after: magictoken.ex_typed_dict_njit.begin
    :end-before: magictoken.ex_typed_dict_njit.end
    :dedent: 12
@@ -730,6 +793,75 @@ range of possible failures. However, the dictionary can be safely read from
 multiple threads as long as the contents of the dictionary do not
 change during the parallel access.
 
+.. _feature-dict-initial-value:
+
+Initial Values
+''''''''''''''
+.. warning::
+  This is an experimental feature!
+
+Typed dictionaries that:
+
+* Are constructed using the curly braces syntax
+* Have literal string keys
+* Have values of a literal type
+
+will have their initial value stored in the ``.initial_value`` property on the
+type so as to permit inspection of these values at compile time. If required,
+to force value based dispatch the :ref:`literally <developer-literally>`
+function will accept a typed dictionary.
+
+Example:
+
+.. literalinclude:: ../../../numba/tests/doc_examples/test_literal_container_usage.py
+   :language: python
+   :caption: from ``test_ex_initial_value_dict_compile_time_consts`` of ``numba/tests/doc_examples/test_literal_container_usage.py``
+   :start-after: magictoken.test_ex_initial_value_dict_compile_time_consts.begin
+   :end-before: magictoken.test_ex_initial_value_dict_compile_time_consts.end
+   :dedent: 12
+   :linenos:
+
+.. _feature-literal-str-key-dict:
+
+Heterogeneous Literal String Key Dictionary
+-------------------------------------------
+
+.. warning::
+  This is an experimental feature!
+
+Numba supports the use of statically declared string key to any value
+dictionaries, for example::
+
+  d = {'a': 1, 'b': 'data', 'c': 2j}
+
+the predominant use of these dictionaries is to orchestrate advanced compilation
+dispatch or as a container for use as a configuration object. The dictionaries
+appear as a ``LiteralStrKeyDict`` type which inherits from ``Literal``, as a
+result the literal values of the keys and the types of the items are available
+at compile time. For example:
+
+.. literalinclude:: ../../../numba/tests/doc_examples/test_literal_container_usage.py
+   :language: python
+   :caption: from ``test_ex_literal_dict_compile_time_consts`` of ``numba/tests/doc_examples/test_literal_container_usage.py``
+   :start-after: magictoken.test_ex_literal_dict_compile_time_consts.begin
+   :end-before: magictoken.test_ex_literal_dict_compile_time_consts.end
+   :dedent: 12
+   :linenos:
+
+Important things to note about these kinds of dictionaries:
+
+#. They are immutable, use of mutating methods e.g. ``.pop()`` will result in
+   compilation failure. Read-only static access and read only methods are
+   supported e.g. ``len()``.
+#. Dynamic access of items is not possible, e.g. ``some_dictionary[x]``, for a
+   value ``x`` which is not a compile time constant. This is because it's
+   impossible statically determine the type of the item being accessed.
+#. Inside the compiler, these dictionaries are actually just named tuples with
+   some extra things added to make them look like they are dictionaries.
+#. They cannot be returned to the interpreter from a compiled function.
+#. The ``.keys()``, ``.values()`` and ``.items()`` methods all functionally
+   operate but return tuples opposed to iterables.
+
 None
 ----
 

numba/core/analysis.py

@@ -608,6 +608,6 @@ def find_literally_calls(func_ir, argtypes):
                     first_loc.setdefault(argindex, assign.loc)
     # Signal the dispatcher to force literal typing
     for pos in marked_args:
-        if not isinstance(argtypes[pos], types.Literal):
+        if not isinstance(argtypes[pos], (types.Literal, types.InitialValue)):
             loc = first_loc[pos]
             raise errors.ForceLiteralArg(marked_args, loc=loc)

numba/core/boxing.py

@@ -1095,3 +1095,7 @@ def unbox_meminfo_pointer(typ, obj, c):
 def unbox_typeref(typ, val, c):
     return NativeValue(c.context.get_dummy_value(), is_error=cgutils.false_bit)
 
+
+@box(types.LiteralStrKeyDict)
+def box_LiteralStrKeyDict(typ, val, c):
+    return box_unsupported(typ, val, c)

numba/core/datamodel/models.py

@@ -311,6 +311,7 @@ def __init__(self, dmm, fe_type):
 @register_default(types.DType)
 @register_default(types.RecursiveCall)
 @register_default(types.MakeFunctionLiteral)
+@register_default(types.Poison)
 class OpaqueModel(PrimitiveModel):
     """
     Passed as opaque pointers
@@ -718,6 +719,8 @@ def __init__(self, dmm, fe_type):
         super(ComplexModel, self).__init__(dmm, fe_type, members)
 
 
+@register_default(types.LiteralList)
+@register_default(types.LiteralStrKeyDict)
 @register_default(types.Tuple)
 @register_default(types.NamedTuple)
 @register_default(types.StarArgTuple)

numba/core/interpreter.py

@@ -17,6 +17,15 @@
 from numba.core.byteflow import Flow, AdaptDFA, AdaptCFA
 from numba.core.unsafe import eh
 
+class _UNKNOWN_VALUE(object):
+    """Represents an unknown value, this is for ease of debugging purposes only.
+    """
+
+    def __init__(self, varname):
+        self._varname = varname
+
+    def __repr__(self):
+        return "_UNKNOWN_VALUE({})".format(self._varname)
 
 _logger = logging.getLogger(__name__)
 
@@ -908,7 +917,47 @@ def op_BUILD_CONST_KEY_MAP(self, inst, keys, keytmps, values, res):
         for kval, tmp in zip(keyconsts, keytmps):
             self.store(kval, tmp)
         items = list(zip(map(self.get, keytmps), map(self.get, values)))
-        expr = ir.Expr.build_map(items=items, size=2, loc=self.loc)
+
+        # sort out literal values
+        literal_items = []
+        for v in values:
+            defns = self.definitions[v]
+            if len(defns) != 1:
+                break
+            defn = defns[0]
+            if not isinstance(defn, ir.Const):
+                break
+            literal_items.append(defn.value)
+
+        def resolve_const(v):
+            defns = self.definitions[v]
+            if len(defns) != 1:
+                return _UNKNOWN_VALUE(self.get(v).name)
+            defn = defns[0]
+            if not isinstance(defn, ir.Const):
+                return _UNKNOWN_VALUE(self.get(v).name)
+            return defn.value
+
+        if len(literal_items) != len(values):
+            literal_dict = {x: resolve_const(y) for x, y in
+                            zip(keytup, values)}
+        else:
+            literal_dict = {x:y for x, y in zip(keytup, literal_items)}
+
+        # to deal with things like {'a': 1, 'a': 'cat', 'b': 2, 'a': 2j}
+        # store the index of the actual used value for a given key, this is
+        # used when lowering to pull the right value out into the tuple repr
+        # of a mixed value type dictionary.
+        value_indexes = {}
+        for i, k in enumerate(keytup):
+            value_indexes[k] = i
+
+        expr = ir.Expr.build_map(items=items,
+                                 size=2,
+                                 literal_value=literal_dict,
+                                 value_indexes=value_indexes,
+                                 loc=self.loc)
+
         self.store(expr, res)
 
     def op_GET_ITER(self, inst, value, res):
@@ -975,8 +1024,48 @@ def op_BUILD_SET(self, inst, items, res):
         self.store(expr, res)
 
     def op_BUILD_MAP(self, inst, items, size, res):
-        items = [(self.get(k), self.get(v)) for k, v in items]
-        expr = ir.Expr.build_map(items=items, size=size, loc=self.loc)
+        got_items = [(self.get(k), self.get(v)) for k, v in items]
+
+        # sort out literal values, this is a bit contrived but is to handle
+        # situations like `{1: 10, 1: 10}` where the size of the literal dict
+        # is smaller than the definition
+        def get_literals(target):
+            literal_items = []
+            values = [self.get(v.name) for v in target]
+            for v in values:
+                defns = self.definitions[v.name]
+                if len(defns) != 1:
+                    break
+                defn = defns[0]
+                if not isinstance(defn, ir.Const):
+                    break
+                literal_items.append(defn.value)
+            return literal_items
+
+        literal_keys = get_literals(x[0] for x in got_items)
+        literal_values = get_literals(x[1] for x in got_items)
+
+
+        has_literal_keys = len(literal_keys) == len(got_items)
+        has_literal_values = len(literal_values) == len(got_items)
+
+        value_indexes = {}
+        if not has_literal_keys and not has_literal_values:
+            literal_dict = None
+        elif has_literal_keys and not has_literal_values:
+            literal_dict = {x: _UNKNOWN_VALUE(y[1]) for x, y in
+                            zip(literal_keys, got_items)}
+            for i, k in enumerate(literal_keys):
+                value_indexes[k] = i
+        else:
+            literal_dict = {x: y for x, y in zip(literal_keys, literal_values)}
+            for i, k in enumerate(literal_keys):
+                value_indexes[k] = i
+
+        expr = ir.Expr.build_map(items=got_items, size=size,
+                                 literal_value=literal_dict,
+                                 value_indexes=value_indexes,
+                                 loc=self.loc)
         self.store(expr, res)
 
     def op_STORE_MAP(self, inst, dct, key, value):

numba/core/ir.py

@@ -444,10 +444,11 @@ def build_set(cls, items, loc):
         return cls(op=op, loc=loc, items=items)
 
     @classmethod
-    def build_map(cls, items, size, loc):
+    def build_map(cls, items, size, literal_value, value_indexes, loc):
         assert isinstance(loc, Loc)
         op = 'build_map'
-        return cls(op=op, loc=loc, items=items, size=size)
+        return cls(op=op, loc=loc, items=items, size=size,
+                   literal_value=literal_value, value_indexes=value_indexes)
 
     @classmethod
     def pair_first(cls, value, loc):

numba/core/lowering.py

@@ -1217,10 +1217,20 @@ def lower_expr(self, resty, expr):
         elif expr.op == "build_list":
             itemvals = [self.loadvar(i.name) for i in expr.items]
             itemtys = [self.typeof(i.name) for i in expr.items]
-            castvals = [self.context.cast(self.builder, val, fromty,
-                                          resty.dtype)
-                        for val, fromty in zip(itemvals, itemtys)]
-            return self.context.build_list(self.builder, resty, castvals)
+            if isinstance(resty, types.LiteralList):
+                castvals = [self.context.cast(self.builder, val, fromty, toty)
+                            for val, toty, fromty in zip(itemvals, resty.types,
+                                                         itemtys)]
+                tup = self.context.make_tuple(self.builder,
+                                              types.Tuple(resty.types),
+                                              castvals)
+                self.incref(resty, tup)
+                return tup
+            else:
+                castvals = [self.context.cast(self.builder, val, fromty,
+                                              resty.dtype)
+                            for val, fromty in zip(itemvals, itemtys)]
+                return self.context.build_list(self.builder, resty, castvals)
 
         elif expr.op == "build_set":
             # Insert in reverse order, as Python does
@@ -1297,7 +1307,6 @@ def storevar(self, value, name):
         Store the value into the given variable.
         """
         fetype = self.typeof(name)
-
         # Define if not already
         self._alloca_var(name, fetype)