Large amount of documentation about functional specialization and ATe…

…rm matching

blaze/__init__.py

@@ -2,7 +2,7 @@
 
 # Install the Blaze library of dispatch functions, must be called
 # early
-import lib
+from lib import *
 
 from datashape import dshape
 from table import Array, Table, NDArray, NDTable

blaze/error.py

@@ -24,6 +24,10 @@ def __init__(self, aterm):
     def __str__(self):
         return "No implementation for '%r'" % self.aterm
 
+# for the RTS
+class InvalidLibraryDefinton(Exception):
+    pass
+
 class NotNumpyCompatible(Exception):
     """
     Raised when we try to convert a datashape into a NumPy dtype

blaze/expr/caterm.pyx

@@ -113,7 +113,7 @@ cdef class PyATerm:
         if isinstance(pattern, basestring):
             a = ATreadFromString(pattern)
             if a == ATEmpty:
-                raise Exception('Invalid ATerm: %s' % pattern)
+                raise InvalidATerm(pattern)
             else:
                 self.a = a
                 self._repr = ATwriteToString(self.a)
@@ -256,12 +256,23 @@ cdef class PyATerm:
     def __repr__(self):
         return "aterm('%s')" % ATwriteToString(self.a)
 
+
+#------------------------------------------------------------------------
+# Exceptions
+#------------------------------------------------------------------------
+
+class InvalidATerm(SyntaxError):
+    pass
+
+class NoAnnotation(KeyError):
+    pass
+
 #------------------------------------------------------------------------
 # Error Handling
 #------------------------------------------------------------------------
 
 cdef void error(char *format, va_list args) with gil:
-    raise Exception(format)
+    raise InvalidATerm(format)
 
 # execute at module init
 cdef ATerm bottomOfStack
@@ -272,16 +283,6 @@ ATsetErrorHandler(error)
 ATsetWarningHandler(error)
 ATsetAbortHandler(error)
 
-#------------------------------------------------------------------------
-# Exceptions
-#------------------------------------------------------------------------
-
-class InvalidATerm(SyntaxError):
-    pass
-
-class NoAnnotation(KeyError):
-    pass
-
 #------------------------------------------------------------------------
 # Constants
 #------------------------------------------------------------------------

blaze/lib.py

@@ -2,11 +2,14 @@
 from math import sqrt
 
 from blaze.datashape import dshape
-from blaze.rts.ffi import PythonFn, install, lift
+from blaze.rts.funcs import PythonFn, install, lift
 from blaze.engine import executors
 from numexpr import evaluate
 from blaze.ts.ucr_dtw import ucr
 
+from blaze.expr.ops import array_like
+from blaze.metadata import aligned
+
 # evaluating this function over a term has a cost, in the future this
 # might be things like calculations for FLOPs associated with the
 # operation. Could be a function of the shape of the term Most of the
@@ -15,25 +18,40 @@
 zerocost = lambda term: 0
 
 #------------------------------------------------------------------------
-# Preinstalled Functions
+# Function Library
 #------------------------------------------------------------------------
 
-# These are side-effectful functions which install the core
-# functions into the RTS dispatcher.
-
-# The signature for PythonF
-
-#   :signature: ATerm pattern matching signature
-
-#   :fn: Python callable instance
-
-#   :mayblock: Whether calling this function may block a thread.
-#              ( i.e. it waits on a disk or socket )
 
-
-# TODO: right now these consume everything but later we'll add functions
-# which specialize on metadata for contigious, chunked, streams,
-# etc...
+# Anatomy of a Blaze Function Def
+# -------------------------------
+
+#   ATerm Pattern Matcher
+#            |                      +-- Type Signature
+#            |                      |
+#            v                      v
+# @lift('Mul(<term>,<term>)', '(a,a) -> a', {
+#     'types'   : {'a': array_like},     <- Type Constraints
+#     'metadata': {'a': aligned, local}, <- Metadata Constraint
+# }, costfn)
+# def multiply(a, b):
+#     return np.multipy(a, b)
+#    |                     |
+#    +---------------------+
+#    Function Implementation
+#
+#
+# If we were to "read" this definition in English. It would read:
+#
+#  > Here is a function called Mul, it matches any expression of two
+#  > graph nodes. In addition the two graph nodes must be of the same
+#  > datashapes (a) or unify/broadcast to the same type (a,a). It
+#  > returns a result of the same type of the operands. In addition the
+#  > two arguments must have metadata annotating them both as aligned
+#  > memory and in system local memory. If the expression in question
+#  > matches these criterion than I can tell you that you can perfom
+#  > this operation in (n*m) FLOPS where n and m are the size of your
+#  > input array, if you can't find a better implementaion than that use
+#  > this function implementation!
 
 @lift('Sqrt(<int>)', 'a -> float32')
 def pyadd(a):
@@ -43,30 +61,30 @@ def pyadd(a):
 def dtw(d1, d2, s, n):
     return ucr.dtw(d1, d2, s, n)
 
-
-install(
-    'Add(<term>,<term>)',
-    PythonFn(np.add.types, np.add, False),
-    zerocost
-)
-
-install(
-    'Mul(<term>,<term>)',
-    PythonFn(np.multiply.types, np.multiply, False),
-    zerocost
-)
-
-install(
-    'Pow(<term>,<term>)',
-    PythonFn(np.power.types, np.power, False),
-    zerocost
-)
-
-install(
-    'Abs(<term>)',
-    PythonFn(np.abs.types, np.abs, False),
-    zerocost
-)
+@lift('Add(<term>,<term>)', '(a,a) -> a')
+def add(a, b):
+    return np.add(a, a)
+
+@lift('Mul(<term>,<term>)', '(a,a)-> a', {
+    'types'   : {'a': array_like},
+    'metadata': {},
+})
+def multiply(a, b):
+    return np.multipy(a, b)
+
+@lift('Pow(<term>,<term>)', '(a,a) -> a', {
+    'types'   : {'a': array_like},
+    'metadata': {},
+})
+def power(a, b):
+    return np.power(a, b)
+
+@lift('Abs(<term>', 'a -> a', {
+    'types'   : {'a': array_like},
+    'metadata': {},
+})
+def abs(a, b):
+    return np.abs(a, b)
 
 # ==============
 # --- Future ---

blaze/expr/metadata.py → blaze/metadata.py

@@ -1,5 +1,5 @@
 from collections import Mapping, OrderedDict
-from utils import Symbol as S
+from blaze.expr.utils import Symbol as S
 
 # The set of possible facets that are specifiable in the
 # metadata. Most structures will define a subset of these. This

blaze/plan.py

@@ -188,7 +188,7 @@ def _Arithmetic(self, term):
         # Returns either a ExternalF ( reference to a external C
         # library ) or a PythonF, a Python callable. These can be
         # anything, numpy ufuncs, numexpr, pandas, cmath whatever
-        from blaze.rts.ffi import lookup
+        from blaze.rts.funcs import lookup
 
         # visit the innermost arguments, push those arguments on
         # the instruction list first

blaze/rts/ffi.py → blaze/rts/funcs.py

@@ -33,6 +33,7 @@
 
 from functools import wraps
 from threading import local
+from blaze.error import InvalidLibraryDefinton
 from inspect import getargspec
 from ctypes import CFUNCTYPE
 
@@ -92,7 +93,7 @@ def lookup(self, aterm):
 
 zerocost = lambda term: 0
 
-def lift(signature, typesig, **params):
+def lift(signature, typesig, constraints=None, **params):
     """ Lift a Python callable into Blaze with the given
     signature. Splice a function graph node constructor in its
     place.
@@ -104,9 +105,16 @@ def lift(signature, typesig, **params):
 
     def outer(pyfn):
         assert callable(pyfn), "Lifted function must be callable"
+        fname = pyfn.func_name
+
+        try:
+            caterm.aterm(signature)
+        except caterm.InvalidATerm as e:
+            raise InvalidLibraryDefinton(*e.args + (fname,))
+
+        # #effectful
         libcall = PythonFn(signature, pyfn)
         install(signature, libcall)
-        fname = pyfn.func_name
 
         sig = getargspec(pyfn)
         nargs = len(sig.args)
@@ -119,11 +127,12 @@ def outer(pyfn):
         cod = params.pop('cod', dynamic)
 
         return type(pyfn.func_name, (Fun,), {
-            'nargs'   : nargs,
-            'fn'      : pyfn,
-            'fname'   : fname,
-            'typesig' : typesig,
-            'cod'     : cod
+            'nargs'       : nargs,
+            'fn'          : pyfn,
+            'fname'       : fname,
+            'typesig'     : typesig,
+            'cod'         : cod,
+            'constraints' : constraints,
         })
 
     return outer

blaze/rts/tests/test_dispatch.py

@@ -1,18 +1,19 @@
-import numpy as np
-
 from blaze.expr.ops import array_like
 from blaze.expr.paterm import ATerm, AAppl, AInt
-from blaze.rts.ffi import install, lift, lookup
+from blaze.rts.funcs import install, lift, lookup
 
-from unittest import skip
+from blaze import add, multiply
 
-def test_install():
+from unittest import skip
 
+def test_match1():
     expr = AAppl(ATerm('Add'), [AInt(1), AInt(2)])
     fn, cost = lookup(expr)
 
-    assert fn.fn == np.add
-    assert cost == 0
+    assert fn.fn == add.fn.im_func
+
+def test_match2():
+    expr = AAppl(ATerm('Mul'), [ATerm('Array'), ATerm('Array')])
+    fn, cost = lookup(expr)
 
-if __name__ == '__main__':
-    test_install()
+    assert fn.fn == multiply.fn.im_func