Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

merged from origin

  • Loading branch information...
commit 9382c4265474c4e91f87b54223097d2d0f4d8b8a 1 parent a5a2f27
@ovillellas ovillellas authored
Showing with 2,699 additions and 1,079 deletions.
  1. +1 −2  .gitignore
  2. +7 −5 Makefile
  3. +10 −9 README.md
  4. +1 −1  bin/blirc
  5. +10 −1 blaze/blir/__init__.py
  6. +15 −0 blaze/blir/btypes.py
  7. +6 −10 blaze/blir/cfg.py
  8. +163 −93 blaze/blir/codegen.py
  9. +8 −1 blaze/blir/errors.py
  10. +92 −24 blaze/blir/exc.py
  11. +51 −0 blaze/blir/intrinsics.py
  12. +1 −1  blaze/blir/opcodes.py
  13. +19 −9 blaze/blir/passes.py
  14. +16 −10 blaze/blir/prelude.c
  15. +14 −3 blaze/blir/typecheck.py
  16. +23 −29 blaze/carray/carrayExtension.pyx
  17. +4 −2 blaze/carray/tests/test_carray.py
  18. +19 −19 blaze/carray/tests/test_large_carray.py
  19. 0  blaze/cgen/__init__.py
  20. +572 −0 blaze/cgen/astprint.py
  21. +104 −0 blaze/cgen/blirgen.py
  22. +0 −122 blaze/cgen/c.py
  23. +87 −0 blaze/cgen/combinators.py
  24. +0 −24 blaze/cgen/gen.py
  25. +260 −0 blaze/cgen/kernels.py
  26. +0 −43 blaze/cgen/py.py
  27. +38 −0 blaze/cgen/utils.py
  28. +2 −0  blaze/datashape/__init__.py
  29. +49 −44 blaze/datashape/coretypes.py
  30. +7 −3 blaze/datashape/parser.py
  31. 0  blaze/include/__init__.py
  32. +1 −1  blaze/module/parser.py
  33. +3 −3 blaze/module/prims.py
  34. +3 −3 blaze/pyfunctions.py
  35. +0 −202 blaze/rts/executors.pyx
  36. +0 −46 blaze/rts/osthread.pyx
  37. +30 −30 blaze/rts/runtime.c
  38. +3 −0  blaze/rts/wrapper.pxd
  39. +94 −0 blaze/rts/wrapper.pyx
  40. +27 −0 blaze/tests/test_cgen1.py
  41. +61 −0 blaze/tests/test_cgen2.py
  42. +8 −2 blaze/tests/test_constructors.py
  43. +5 −5 blaze/tests/test_datashape.py
  44. +36 −0 blaze/tests/test_datashape_creation.py
  45. +13 −1 blaze/tests/test_newparser.py
  46. +24 −21 blaze/tests/test_numpy_compat.py
  47. +1 −1  blaze/tests/test_parser.py
  48. +175 −10 docs/source/compiler.rst
  49. +2 −0  docs/source/datashape-spec.rst
  50. +7 −148 docs/source/datashape.rst
  51. +2 −5 docs/source/eclass.rst
  52. +9 −4 docs/source/index.rst
  53. +108 −14 docs/source/module.rst
  54. +0 −20 docs/source/prims.rst
  55. +225 −73 docs/source/svg/pipeline.svg
  56. +1 −4 docs/source/typeinference.rst
  57. +36 −0 samples/blir/black_scholes.bl
  58. +22 −19 samples/blir/dot.bl
  59. +28 −0 samples/blir/fibonacci.bl
  60. +72 −0 samples/blirchunk.py
  61. +8 −4 samples/numpy_blir.py
  62. +2 −2 samples/persist_example.py
  63. +17 −6 setup.py
  64. +97 −0 test_iterator.py
View
3  .gitignore
@@ -74,9 +74,8 @@ blaze/engine/*.[ch]
blaze/expr/*.[ch]
blaze/desc/*.[ch]
blaze/algo/*.[ch]
-blaze/rts/executors.c
+blaze/rts/runtime.c
blaze/ts/ucr_dtw/ucr.c
-blaze/rts/executors.c
# Sphinx Docs
docs/_build
View
12 Makefile
@@ -1,15 +1,17 @@
.PHONY: all docs tests build clean web
CC = gcc
-LPYTHON = $(shell python-config --includes --libs)
-CFLAGS = -shared -fPIC -lpthread $(LPYTHON)
+LPYTHON = $(shell python-config --includes)
+LNUMPY = $(shell python -c "import numpy; print '-I' + numpy.get_include()")
-all: build
+CFLAGS = -lpthread $(LPYTHON) $(LNUMPY)
+
+all: build blir
# stupid hack for now
blir:
- $(CC) $(CFLAGS) blaze/blir/datashape.c -o blaze/blir/datashape.o
- $(CC) $(CFLAGS) blaze/blir/prelude.c blaze/blir/datashape.o -o blaze/blir/prelude.dylib
+ $(CC) $(CFLAGS) -shared -fPIC blaze/blir/datashape.c -o blaze/blir/datashape.o
+ $(CC) $(CFLAGS) -shared -fPIC blaze/blir/datashape.o blaze/blir/prelude.c -o blaze/blir/prelude.so
build:
python setup.py build_ext --inplace
View
19 README.md
@@ -15,9 +15,9 @@ Our goal is to allow easy composition of low level computation kernels
( C, Fortran, Numba ) to form complex data transformations on large
datasets.
-In Blaze computations are described in a high-level language ( Python
-) but executed on a low-level runtime outside of Python. Allowing the
-easy mapping of high-level expertise to data while not sacrificing
+In Blaze, computations are described in a high-level language ( Python
+) but executed on a low-level runtime outside of Python, enabling the
+easy mapping of high-level expertise to data without sacrificing
low-level performance. Blaze aims to bring Python and NumPy into the
massively-multicore arena, allowing it to able to leverage many CPU and
GPU cores across computers, virtual machines and cloud services.
@@ -35,7 +35,7 @@ innovations in data management, analytics, and distributed computation.
Blaze aims to be a "glue project" allowing many different users of
other PyData projects ( Pandas, Theano, Numba, SciPy, Scikit-Learn)
-to interoperate at the appication level and at the library level with
+to interoperate at the application level and at the library level with
the goal of being able to to lift their existing functionality into a
distributed context.
@@ -47,7 +47,7 @@ Status
------
Blaze is a work in progress at the moment. The code is quite a distance
-from feature complete. The code is released in an effort to start a
+from feature-complete. The code is released in an effort to start a
public discussion with our end users and community.
Documentation
@@ -97,8 +97,9 @@ $ python -m blaze.testing
Alternative Installation
------------------------
-If you desire not to use Anaconda it is possible to build Blaze using
-standard Python tools. This method is not recommended.
+If for some reason you wish to avoid using Anaconda
+it is possible to build Blaze using standard Python tools.
+This method is not recommended.
1) After you have checked out the Blaze source, create a virtualenv
under the root of the Blaze repo.
@@ -156,8 +157,8 @@ Contributing
------------
Anyone wishing to discuss on Blaze should join the
-[blaze-dev](https://groups.google.com/a/continuum.io/forum/#!forum/blaze
--dev) mailing list at: blaze-dev@continuum.io
+[blaze-dev](https://groups.google.com/a/continuum.io/forum/#!forum/blaze-dev)
+mailing list at: blaze-dev@continuum.io
License
-------
View
2  bin/blirc
@@ -1,3 +1,3 @@
#!/usr/bin/env bash
-PYTHONPATH=. python -m blaze.blir.passes $@
+python -m blaze.blir.passes $@
View
11 blaze/blir/__init__.py
@@ -1,2 +1,11 @@
from .passes import compile, CompileError
-from .exc import execute
+from .exc import Context, execute
+from .errors import log
+
+def bitcode(env):
+ """ Print LLVM bitcode for the given compiled environment """
+ return env['cgen'].module
+
+def assembly(env):
+ """ Print x86 assembly for the given compiled environment """
+ return env['cgen'].module.to_native_assembly()
View
15 blaze/blir/btypes.py
@@ -38,6 +38,12 @@ def __init__(self, name, kind, zero, binary_ops,unary_ops,cmp_ops):
self.unary_ops = unary_ops
self.cmp_ops = cmp_ops
+ def __eq__(self, other):
+ if isinstance(other, Type):
+ return self.name == other.name
+ else:
+ return False
+
def __repr__(self):
return '<%s>' % self.name
@@ -48,6 +54,15 @@ def __init__(self, cons, arg):
self.cons = cons
self.arg = arg
+ def __eq__(self, other):
+ if isinstance(other, PType):
+ return self.cons.name == other.cons.name
+ else:
+ return False
+
+ def __ne__(self, other):
+ return not (self == other)
+
def __repr__(self):
return '<%s[%s]>' % (self.cons.name, self.arg.name)
View
16 blaze/blir/cfg.py
@@ -60,19 +60,17 @@ def atemp(self, val):
# f%d - floats
# s%d - string
# b%d - bool
- name = TEMP_NAMING % (val.name[0], self.assignments[val.name])
+ name = TEMP_NAMING % (val.name, self.assignments[val.name])
self.assignments[val.name] += 1
return name
def visit_Module(self, node):
self.visit(node.body)
- # --
self.block.append(('RETURN', btypes.void_type))
def visit_UnaryOp(self, node):
self.visit(node.expr)
- # --
target = self.atemp(node.expr.type)
opcode = unary_ops[node.op]
@@ -84,7 +82,6 @@ def visit_UnaryOp(self, node):
def visit_BinOp(self, node):
self.visit(node.left)
self.visit(node.right)
- # --
target = self.atemp(node.type)
opcode = binary_ops[node.op]
@@ -109,7 +106,6 @@ def visit_Const(self,node):
def visit_Assign(self,node):
self.visit(node.expr)
self.visit(node.store_location)
- # --
def visit_LoadVariable(self, node):
target = self.atemp(node.type)
@@ -178,7 +174,7 @@ def visit_VarDecl(self, node):
inst = ('STORE', node.expr.ssa_name, node.name)
self.block.append(inst)
- def visit_ConstDecl(self,node):
+ def visit_ConstDecl(self, node):
if node.is_global:
inst = ('GLOBAL', node.expr.type, node.name)
else:
@@ -190,12 +186,12 @@ def visit_ConstDecl(self,node):
inst = ('STORE', node.expr.ssa_name, node.name)
self.block.append(inst)
- def visit_ExternFuncDecl(self,node):
+ def visit_ExternFuncDecl(self, node):
self.visit(node.sig)
# --
- args = tuple(arg.type.name for arg in node.sig.parameters)
- inst = ('CALL_FOREIGN', node.sig.name, node.sig.type.name) + args
+ args = tuple(arg.type for arg in node.sig.parameters)
+ inst = ('DEF_FOREIGN', node.sig.name, node.sig.type, args)
self.block.append(inst)
def visit_FunctionCall(self, node):
@@ -205,7 +201,7 @@ def visit_FunctionCall(self, node):
args.append(arg.ssa_name)
target = self.atemp(node.type)
- inst = ('CALL_FUNCTION', node.name) + tuple(args) + (target,)
+ inst = ('CALL_FUNCTION', node.name, args, target)
self.block.append(inst)
node.ssa_name = target
View
256 blaze/blir/codegen.py
@@ -1,6 +1,6 @@
import btypes
+import ctypes
import errors
-from collections import defaultdict
import llvm.core as lc
import llvm.passes as lp
@@ -8,10 +8,14 @@
from llvm import LLVMException
from llvm.core import Module, Builder, Function, Type, Constant, GlobalVariable
+from collections import defaultdict
+
#------------------------------------------------------------------------
# LLVM Types
#------------------------------------------------------------------------
+ptrsize = ctypes.sizeof(ctypes.c_void_p)
+
int_type = Type.int()
float_type = Type.double()
bool_type = Type.int(1)
@@ -20,10 +24,9 @@
pointer = Type.pointer
+any_type = pointer(Type.int(ptrsize))
string_type = pointer(char_type)
-data_type = pointer(Type.int(8)) # char*
-
# { i32*, i32, i32* }
array_type = lambda elt_type: Type.struct([
pointer(elt_type), # data | (<type>)*
@@ -34,10 +37,13 @@
# opaque for now
blaze_type = lambda datashape: Type.opaque(name="blaze")
-any_type = Type.void()
+#------------------------------------------------------------------------
+# Constants
+#------------------------------------------------------------------------
false = Constant.int(bool_type, 0)
true = Constant.int(bool_type, 1)
+zero = Constant.int(int_type, 0)
#------------------------------------------------------------------------
# Type Relations
@@ -76,7 +82,7 @@ def arg_typemap(ty):
'!=' : lc.FCMP_ONE
}
-sing_instrs = {
+int_instrs = {
'>' : lc.ICMP_SGT,
'<' : lc.ICMP_SLT,
'==' : lc.ICMP_EQ,
@@ -90,7 +96,7 @@ def arg_typemap(ty):
'!=' : lc.ICMP_NE
}
-bitwise = { '&&', '||' }
+logic_instrs = { '&&', '||' }
#------------------------------------------------------------------------
# Prelude
@@ -101,8 +107,29 @@ def arg_typemap(ty):
'show_float' : Type.function(void_type, [float_type], False),
'show_bool' : Type.function(void_type, [bool_type], False),
'show_string' : Type.function(void_type, [string_type], False),
+ 'show_array' : Type.function(void_type, [any_type], False),
}
+
+def build_intrinsics(mod):
+ # XXX define in seperate module and then link in
+ import intrinsics
+
+ ins = {}
+
+ for name, intr in intrinsics.llvm_intrinsics.iteritems():
+ # get the function signature
+ name, retty, argtys = getattr(intrinsics, name)
+
+ largtys = map(arg_typemap, argtys)
+ #lretty = arg_typemap(retty)
+
+ lfunc = Function.intrinsic(mod, intr, largtys)
+ ins[name] = lfunc
+
+ #mod.verify()
+ return mod, ins
+
#------------------------------------------------------------------------
# Function Level Codegen
#------------------------------------------------------------------------
@@ -111,6 +138,8 @@ def arg_typemap(ty):
MODULE_NAMING = '.module.%x'
class LLVMEmitter(object):
+ """ LLVM backend for Blir opcodes. """
+
def __init__(self, name="blirkernel"):
self.module = Module.new(name)
@@ -121,24 +150,29 @@ def __init__(self, name="blirkernel"):
self.globals = {}
self.locals = {}
- self.arrays = defaultdict(dict)
- self.temps = {}
+ self.stack = {} # stack allocated referred by ssa ref
+ self.refs = defaultdict(dict) # opaque referencse
self.intrinsics = {}
- self.define_intrinsics()
+ self.add_prelude()
+
+ intmod, intrinsics = build_intrinsics(self.module)
+
+ #self.module.link_in(intmod)
+ self.globals.update(intrinsics)
def visit_op(self, instr):
for op in instr:
- opcode = op[0]
+ op_code = op[0]
op_args = op[1:]
- if hasattr(self, "op_"+opcode):
- getattr(self, "op_"+opcode)(*op_args)
+ if hasattr(self, "op_"+op_code):
+ getattr(self, "op_"+op_code)(*op_args)
else:
- raise Exception("Can't translate opcode: op_"+opcode)
+ raise Exception("Can't translate opcode: op_"+op_code)
- def start_function(self, name, retty, argtypes):
+ def start_function(self, name, retty, argtys):
rettype = arg_typemap(retty)
- argtypes = [arg_typemap(arg) for arg in argtypes]
+ argtypes = [arg_typemap(arg) for arg in argtys]
func_type = Type.function(rettype, argtypes, False)
self.function = Function.new(self.module, func_type, name)
@@ -148,12 +182,11 @@ def start_function(self, name, retty, argtypes):
self.exit_block = self.function.append_basic_block("exit")
self.locals = {}
- self.temps = {}
+ self.stack = {}
if rettype is not void_type:
self.locals['retval'] = self.builder.alloca(rettype, "retval")
- # allow recursive calling
self.globals[name] = self.function
def end_function(self):
@@ -165,7 +198,7 @@ def end_function(self):
else:
self.builder.ret_void()
- def define_intrinsics(self):
+ def add_prelude(self):
for name, function in prelude.iteritems():
self.intrinsics[name] = Function.new(
self.module,
@@ -181,7 +214,7 @@ def set_block(self, block):
self.builder.position_at_end(block)
def cbranch(self, cond, true_block, false_block):
- self.builder.cbranch(self.temps[cond], true_block, false_block)
+ self.builder.cbranch(self.stack[cond], true_block, false_block)
def branch(self, next_block):
self.builder.branch(next_block)
@@ -204,6 +237,7 @@ def const(self, val):
def lookup_var(self, name):
# XXX replace this with just passing the symbol table
+ # from the annotator
if name in self.locals:
return self.locals[name]
else:
@@ -216,6 +250,8 @@ def puts(self, val):
self.call('print_float', [val])
elif val.type == string_type:
self.call('print_string', [val])
+ elif val.type == array_type:
+ self.call('print_array', [val])
else:
raise NotImplementedError
@@ -225,11 +261,11 @@ def puts(self, val):
def op_LOAD_CONST(self, value, target):
if isinstance(value, bool):
- self.temps[target] = Constant.int(bool_type, value)
+ self.stack[target] = Constant.int(bool_type, value)
elif isinstance(value, int):
- self.temps[target] = Constant.int(int_type, value)
+ self.stack[target] = Constant.int(int_type, value)
elif isinstance(value, (float, long)):
- self.temps[target] = Constant.real(float_type, value)
+ self.stack[target] = Constant.real(float_type, value)
elif isinstance(value, str):
# create null terminated string
n = 0
@@ -248,7 +284,7 @@ def op_LOAD_CONST(self, value, target):
globalstr = self.module.add_global_variable(content.type, name)
globalstr.initializer = content
globalstr.linkage = lc.LINKAGE_LINKONCE_ODR
- self.temps[target] = globalstr.bitcast(pointer(content.type.element))
+ self.stack[target] = globalstr.bitcast(pointer(content.type.element))
else:
raise NotImplementedError
@@ -264,13 +300,27 @@ def op_GLOBAL(self, ty, name):
self.globals[name] = var
def op_LOAD(self, name, target):
- self.temps[target] = self.builder.load(self.lookup_var(name), target)
+ var = self.lookup_var(name)
+
+ # aggregate type ( reference )
+ if name in self.refs:
+ ref = self.refs[name]['_struct']
+ ptr = self.builder.gep(ref, [zero])
+ self.stack[target] = self.builder.bitcast(ptr, any_type)
+
+ # any type ( reference )
+ elif var.type == any_type:
+ self.stack[target] = var
+
+ # simple type ( value )
+ else:
+ self.stack[target] = self.builder.load(var, target)
def op_STORE(self, source, target):
- self.builder.store(self.temps[source], self.lookup_var(target))
+ self.builder.store(self.stack[source], self.lookup_var(target))
def op_ARRAYLOAD(self, source, index, target, cc=False):
- arr = self.arrays[source]
+ arr = self.refs[source]
data_ptr = arr['data']
if cc:
@@ -278,7 +328,7 @@ def op_ARRAYLOAD(self, source, index, target, cc=False):
for i, idx in enumerate(index.elts):
ic = self.const(i)
- idxv = self.temps[idx.ssa_name]
+ idxv = self.stack[idx.ssa_name]
stride = arr['strides']
stride_elt = self.builder.load(self.builder.gep(stride, [ic]))
@@ -288,14 +338,14 @@ def op_ARRAYLOAD(self, source, index, target, cc=False):
)
else:
- offset = self.temps[index]
+ offset = self.stack[index]
val = self.builder.gep(data_ptr, [offset])
elt = self.builder.load(val)
- self.temps[target] = elt
+ self.stack[target] = elt
def op_ARRAYSTORE(self, source, index, target, cc=False):
- arr = self.arrays[source]
+ arr = self.refs[source]
data_ptr = arr['data']
if cc:
@@ -303,7 +353,7 @@ def op_ARRAYSTORE(self, source, index, target, cc=False):
for i, idx in enumerate(index.elts):
ic = self.const(i)
- idxv = self.temps[idx.ssa_name]
+ idxv = self.stack[idx.ssa_name]
stride = arr['strides']
stride_elt = self.builder.load(self.builder.gep(stride, [ic]))
@@ -312,94 +362,96 @@ def op_ARRAYSTORE(self, source, index, target, cc=False):
self.builder.mul(stride_elt, idxv)
)
else:
- offset = self.temps[index]
+ offset = self.stack[index]
val = self.builder.gep(data_ptr, [offset])
- self.builder.store(self.temps[target], val)
+ self.builder.store(self.stack[target], val)
def op_BINARY_ADD(self, ty, left, right, val):
- lv = self.temps[left]
- rv = self.temps[right]
+ lv = self.stack[left]
+ rv = self.stack[right]
if ty == btypes.int_type:
- self.temps[val] = self.builder.add(lv, rv, val)
+ self.stack[val] = self.builder.add(lv, rv, val)
elif ty == btypes.float_type:
- self.temps[val] = self.builder.fadd(lv, rv, val)
+ self.stack[val] = self.builder.fadd(lv, rv, val)
def op_BINARY_SUBTRACT(self, ty, left, right, val):
- lv = self.temps[left]
- rv = self.temps[right]
+ lv = self.stack[left]
+ rv = self.stack[right]
if ty == btypes.int_type:
- self.temps[val] = self.builder.sub(lv, rv, val)
+ self.stack[val] = self.builder.sub(lv, rv, val)
elif ty == btypes.float_type:
- self.temps[val] = self.builder.fsub(lv, rv, val)
+ self.stack[val] = self.builder.fsub(lv, rv, val)
def op_BINARY_MULTIPLY(self, ty, left, right, val):
- lv = self.temps[left]
- rv = self.temps[right]
+ lv = self.stack[left]
+ rv = self.stack[right]
if ty == btypes.int_type:
- self.temps[val] = self.builder.mul(lv, rv, val)
+ self.stack[val] = self.builder.mul(lv, rv, val)
elif ty == btypes.float_type:
- self.temps[val] = self.builder.fmul(lv, rv, val)
+ self.stack[val] = self.builder.fmul(lv, rv, val)
def op_BINARY_DIVIDE(self, ty, left, right, val):
- lv = self.temps[left]
- rv = self.temps[right]
+ lv = self.stack[left]
+ rv = self.stack[right]
if ty == btypes.int_type:
- self.temps[val] = self.builder.sdiv(lv, rv, val)
+ self.stack[val] = self.builder.sdiv(lv, rv, val)
elif ty == btypes.float_type:
- self.temps[val] = self.builder.fsdiv(lv, rv, val)
+ self.stack[val] = self.builder.fdiv(lv, rv, val)
#------------------------------------------------------------------------
# Unary Operators
#------------------------------------------------------------------------
def op_UNARY_POSITIVE(self, ty, source, target):
- self.temps[target] = self.temps[source]
+ self.stack[target] = self.stack[source]
def op_UNARY_NEGATIVE(self, ty, source, target):
if ty == btypes.int_type:
- self.temps[target] = self.builder.sub(
+ self.stack[target] = self.builder.sub(
Constant.int(int_type, 0),
- self.temps[source],
+ self.stack[source],
target
)
elif ty == btypes.float_type:
- self.temps[target] = self.builder.fsub(
+ self.stack[target] = self.builder.fsub(
Constant.real(float_type, 0.0),
- self.temps[source],
+ self.stack[source],
target
)
def op_UNARY_NOT(self, ty, source, val):
- self.temps[val] = self.builder.icmp(lc.ICMP_EQ, self.temps[source], false, val)
+ if ty == btypes.int_type:
+ self.stack[val] = self.builder.icmp(lc.ICMP_EQ, self.stack[source], zero, val)
+ elif ty == btypes.bool_type:
+ self.stack[val] = self.builder.not_(self.stack[source])
def op_COMPARE(self, op, ty, left, right, val):
- lv = self.temps[left]
- rv = self.temps[right]
+ lv = self.stack[left]
+ rv = self.stack[right]
if ty == btypes.int_type:
- instr = sing_instrs[op]
- self.temps[val] = self.builder.icmp(instr, lv, rv, val)
+ instr = int_instrs[op]
+ self.stack[val] = self.builder.icmp(instr, lv, rv, val)
elif ty == btypes.float_type:
instr = float_instrs[op]
- self.temps[val] = self.builder.fcmp(instr, lv, rv, val)
+ self.stack[val] = self.builder.fcmp(instr, lv, rv, val)
- # ick
if ty == btypes.bool_type:
- if op in bitwise:
+ if op in logic_instrs:
if op == '&&':
- self.temps[val] = self.builder.and_(lv, rv, val)
+ self.stack[val] = self.builder.and_(lv, rv, val)
elif op == '||':
- self.temps[val] = self.builder.or_(lv, rv, val)
+ self.stack[val] = self.builder.or_(lv, rv, val)
else:
- instr = sing_instrs[op]
- self.temps[val] = self.builder.icmp(instr, lv, rv, val)
+ instr = int_instrs[op]
+ self.stack[val] = self.builder.icmp(instr, lv, rv, val)
#------------------------------------------------------------------------
# Show Functions
@@ -407,33 +459,35 @@ def op_COMPARE(self, op, ty, left, right, val):
def op_PRINT(self, ty, source):
if ty == btypes.int_type:
- self.call('show_int', [self.temps[source]])
+ self.call('show_int', [self.stack[source]])
elif ty == btypes.float_type:
- self.call('show_float', [self.temps[source]])
+ self.call('show_float', [self.stack[source]])
elif ty == btypes.string_type:
- self.call('show_string', [self.temps[source]])
+ self.call('show_string', [self.stack[source]])
elif ty == btypes.bool_type:
- tmp = self.builder.zext(self.temps[source], bool_type)
+ tmp = self.builder.zext(self.stack[source], bool_type)
self.call('show_bool', [tmp])
+ elif isinstance(ty, btypes.PType) and ty.cons.type == btypes.array_type:
+ self.call('show_array', [self.stack[source]])
else:
+ import pdb; pdb.set_trace()
raise NotImplementedError
#------------------------------------------------------------------------
# Extern Functions
#------------------------------------------------------------------------
- def op_CALL_FOREIGN(self, name, retty, *argtys):
- _rettype = typemap[retty]
- _argtys = [typemap[arg] for arg in argtys]
+ def op_DEF_FOREIGN(self, name, retty, argtys):
+ largtys = map(arg_typemap, argtys)
+ lretty = arg_typemap(retty)
- func_type = Type.function(_rettype, _argtys, False)
+ func_type = Type.function(lretty, largtys, False)
self.globals[name] = Function.new(self.module, func_type, name)
- def op_CALL_FUNCTION(self, funcname, *args):
- target = args[-1]
+ def op_CALL_FUNCTION(self, funcname, args, target):
func = self.globals[funcname]
- argvals = [self.temps[name] for name in args[:-1]]
- self.temps[target] = self.builder.call(func, argvals)
+ argvals = [self.stack[name] for name in args]
+ self.stack[target] = self.builder.call(func, argvals)
#------------------------------------------------------------------------
# Return
@@ -443,7 +497,7 @@ def op_RETURN(self, ty, val=None):
if ty == btypes.void_type:
self.builder.branch(self.exit_block)
else:
- self.builder.store(self.temps[val], self.locals['retval'])
+ self.builder.store(self.stack[val], self.locals['retval'])
self.builder.branch(self.exit_block)
#------------------------------------------------------------------------
@@ -452,6 +506,8 @@ def op_RETURN(self, ty, val=None):
def op_LOAD_ARGUMENT(self, ty, name, num):
# aggregate types, pass by reference
+ arg = self.function.args[num]
+
if isinstance(ty, btypes.PType):
# Blaze Types
@@ -463,7 +519,7 @@ def op_LOAD_ARGUMENT(self, ty, name, num):
# NumPy ndarray
# -------------
else:
- struct_ptr = self.function.args[num]
+ struct_ptr = arg
zero = self.const(0)
one = self.const(1)
@@ -473,18 +529,22 @@ def op_LOAD_ARGUMENT(self, ty, name, num):
dims = self.builder.gep(struct_ptr, [zero, one], name=(name + '_dims'))
strides = self.builder.gep(struct_ptr, [zero, two], name=(name + '_strides'))
- self.arrays[name]['_struct'] = struct_ptr
- self.arrays[name]['_dtype'] = typemap[ty.arg.type.name]
- self.arrays[name]['data'] = self.builder.load(data)
- self.arrays[name]['dims'] = self.builder.load(dims)
- self.arrays[name]['strides'] = self.builder.load(strides)
+ self.refs[name]['_struct'] = struct_ptr
+ self.refs[name]['_dtype'] = typemap[ty.arg.type.name]
+ self.refs[name]['data'] = self.builder.load(data)
+ self.refs[name]['dims'] = self.builder.load(dims)
+ self.refs[name]['strides'] = self.builder.load(strides)
- self.locals[name] = self.arrays[name]
+ self.locals[name] = self.refs[name]
+
+ # opaque any types
+ elif arg.type == any_type:
+ self.locals[name] = arg
# concrete types, pass by value
else:
- var = self.builder.alloca(int_type, name)
- self.builder.store(self.function.args[num], var)
+ var = self.builder.alloca(arg.type, name)
+ self.builder.store(arg, var)
self.locals[name] = var
#------------------------------------------------------------------------
@@ -556,13 +616,13 @@ def visit_ForBlock(self, block):
varname = block.var
inc = builder.alloca(int_type, varname)
self.cgen.locals[varname] = inc
- builder.store(self.cgen.temps[start], inc)
+ builder.store(self.cgen.stack[start], inc)
# ------------------------------------------
self.cgen.branch(test_block)
self.cgen.set_block(test_block)
# ------------------------------------------
- stopv = self.cgen.temps[stop]
+ stopv = self.cgen.stack[stop]
cond = builder.icmp(lc.ICMP_SLT, builder.load(inc), stopv)
builder.cbranch(cond, body_block, end_block)
@@ -604,21 +664,32 @@ def visit_WhileBlock(self, block):
class LLVMOptimizer(object):
def __init__(self, module, opt_level=3):
+ INLINER_THRESHOLD = 1000
+
self.fpm = lp.FunctionPassManager.new(module)
self.pmb = lp.PassManagerBuilder.new()
+ self.pm = lp.PassManager.new()
+
self.pmb.opt_level = opt_level
self.pmb.vectorize = True
+ self.pmb.use_inliner_with_threshold(INLINER_THRESHOLD)
+
+ self.pmb.populate(self.pm)
self.pmb.populate(self.fpm)
+ def runmodule(self, module):
+ self.pm.run(module)
+
def run(self, func):
self.fpm.run(func)
- def diff(self, func):
+ def diff(self, func, module):
from difflib import Differ
d = Differ()
before = str(func)
self.run(func)
+ self.runmodule(module)
after = str(func)
diff = d.compare(before.splitlines(), after.splitlines())
@@ -628,7 +699,6 @@ def diff(self, func):
#------------------------------------------------------------------------
def ddump_optimizer(source):
- import lexer
import parser
import cfg
import typecheck
@@ -655,7 +725,7 @@ def ddump_optimizer(source):
optimizer = codegen.LLVMOptimizer(cgen.module)
print 'Optimizer Diff'.center(80, '=')
- optimizer.diff(function)
+ optimizer.diff(function, cgen.module)
#------------------------------------------------------------------------
View
9 blaze/blir/errors.py
@@ -2,6 +2,7 @@
from contextlib import contextmanager
_listeners = []
+_errlog = []
_count = 0
#------------------------------------------------------------------------
@@ -9,13 +10,14 @@
#------------------------------------------------------------------------
def error(lineno, message, filename=None):
- global _count
+ global _count, _errlog
if not filename:
errmsg = "{}: {}".format(lineno, message)
else:
errmsg = "{}:{}: {}".format(filename, lineno, message)
for listener in _listeners:
listener(errmsg)
+ _errlog.append(errmsg)
_count += 1
def _default_handler(msg):
@@ -24,8 +26,13 @@ def _default_handler(msg):
def reset():
global _count
+ global _errlog
_count = 0
+def log():
+ global _errlog
+ return _errlog
+
#------------------------------------------------------------------------
# Toplevel
#------------------------------------------------------------------------
View
116 blaze/blir/exc.py
@@ -1,14 +1,17 @@
-import os
import sys
import time
import array
import ctypes
import numpy as np
from os.path import realpath, dirname, join
-from collections import Iterable
from types import ModuleType
+from bitey.bind import wrap_llvm_module
+
import llvm.ee as le
+from llvm.workaround.avx_support import detect_avx_support
+
+any_type = ctypes.c_char_p
_nptypemap = {
'c': ctypes.c_char,
@@ -35,6 +38,19 @@
# Arguments
#------------------------------------------------------------------------
+class Array(ctypes.Structure):
+ _fields_ = [
+ ("data" , ctypes.c_void_p),
+ ("nd" , ctypes.c_int),
+ ("strides" , ctypes.POINTER(ctypes.c_int)),
+ ]
+
+def fake_array(dataptr):
+ dims = 1
+ _strides = (8,)
+ strides = (ctypes.c_int*dims)(*_strides)
+ return Array(dataptr, 1, strides)
+
def array_decon(na):
ctype = _nptypemap[na.dtype.char]
_strides = ((s/na.dtype.itemsize) for s in na.strides)
@@ -46,8 +62,9 @@ def array_decon(na):
def adapt(arg, val):
"""
- Adapt arguments to run inside ExecutionEngine.
+ Adapt arguments to pass to ExecutionEngine.
"""
+
if isinstance(val, np.ndarray):
ndarray = arg._type_ # auto-generated by bitey
@@ -57,7 +74,7 @@ def adapt(arg, val):
elif isinstance(val, array.array):
raise NotImplementedError
- elif isinstance(val, Iterable):
+ elif isinstance(val, list):
# Build an array from an iterable, not the best thing to
# do performance-wise usually...
ndarray = arg._type_ # auto-generated by bitey
@@ -69,9 +86,19 @@ def adapt(arg, val):
strides = (ctypes.c_int*1)(len(val))
return ndarray(data, dims, strides)
- else:
+
+ elif isinstance(val, (int, long, str, float)):
return val
+ elif isinstance(val, tuple):
+ return arg._type_(*val)
+
+ elif isinstance(val, ctypes.Structure):
+ return ctypes.cast(ctypes.pointer(val), arg._type_)
+
+ else:
+ return ctypes.c_char_p(id(val))
+
def wrap_arguments(fn, args):
if args:
largs = map(adapt, fn.argtypes, args)
@@ -83,37 +110,78 @@ def wrap_arguments(fn, args):
# Toplevel
#------------------------------------------------------------------------
-# mostly just delegates to Bitey because I don't feel like rolling a
-# ctypes wrapper, this is just for debugging anyways so whatever.
-def execute(env, args=None, fname=None, timing=False):
- from bitey.bind import wrap_llvm_module
+class Context(object):
- args = args or ()
+ def __init__(self, env, libs=None):
+ self.destroyed = False
+ libs = libs or ['prelude']
- if os.name == "posix" and sys.platform == "darwin":
- prelude = join(dirname(realpath(__file__)), 'prelude.dylib')
- elif os.name == "posix":
- prelude = join(dirname(realpath(__file__)), 'prelude.so')
- else:
- raise NotImplementedError
+ for lib in libs:
+ if 'darwin' in sys.platform:
+ prelude = join(dirname(realpath(__file__)), lib + '.dylib')
+ elif 'linux' in sys.platform:
+ prelude = join(dirname(realpath(__file__)), lib+ '.so')
+ else:
+ raise NotImplementedError
+
+ # XXX: yeah, don't do this
+ ctypes._dlopen(prelude, ctypes.RTLD_GLOBAL)
+
+ cgen = env['cgen']
+
+ self.__namespace = cgen.globals
+ self.__llmodule = cgen.module.clone()
+
+ if not detect_avx_support():
+ tc = le.TargetMachine.new(features='-avx', cm=le.CM_JITDEFAULT)
+ else:
+ tc = le.TargetMachine.new(features='', cm=le.CM_JITDEFAULT)
+
+ eb = le.EngineBuilder.new(self.__llmodule)
+ self.__engine = eb.create(tc)
+ #self.__engine.run_function(cgen.globals['__module'], [])
+
+ mod = ModuleType('blir_wrapper')
+ wrap_llvm_module(cgen.module, self.__engine, mod)
+
+ self.__mod = mod
- ctypes._dlopen(prelude, ctypes.RTLD_GLOBAL)
+ def lookup_fn(self, fname):
+ return getattr(self.__mod, fname)
- cgen = env['cgen']
- module = cgen.module
+ def lookup_fnptr(self, fname):
+ return self.__engine.get_pointer_to_function(self.__namespace[fname])
- executor = le.ExecutionEngine.new(module)
- executor.run_function(cgen.globals['__module'], [])
+ @property
+ def mod(self):
+ return self.__mod
- mod = ModuleType('mymodule')
- wrap_llvm_module(cgen.module, executor, mod)
+ def destroy(self):
+ if not self.destroyed:
+ del self.__engine
+ else:
+ raise RuntimeError("Context already destroyed")
+
+#------------------------------------------------------------------------
+# Execution
+#------------------------------------------------------------------------
+
+# mostly just delegates to Bitey because I don't feel like rolling a
+# ctypes wrapper, this is just for debugging anyways so whatever.
+def execute(ctx, args=None, fname=None, timing=False):
+
+ args = args or ()
- lfn = getattr(mod, fname or 'main')
+ try:
+ lfn = ctx.lookup_fn(fname or 'main')
+ except AttributeError:
+ raise Exception, 'Compiled module has no toplevel function %s' % fname
largs = wrap_arguments(lfn, args)
if timing:
start = time.time()
+ res = None
if len(lfn.argtypes) == 0:
res = lfn()
elif len(lfn.argtypes) == len(args):
View
51 blaze/blir/intrinsics.py
@@ -0,0 +1,51 @@
+import llvm.core as lc
+
+from btypes import int_type, float_type
+
+#------------------------------------------------------------------------
+# Signatures
+#------------------------------------------------------------------------
+
+# trigonometric
+sin = 'sin', float_type, [float_type]
+cos = 'cos', float_type, [float_type]
+tan = 'tan', float_type, [float_type]
+asin = 'asin', float_type, [float_type]
+acos = 'acos', float_type, [float_type]
+atan = 'atan', float_type, [float_type]
+
+# hyperbolic
+sinh = 'sinh', float_type, [float_type]
+cosh = 'cosh', float_type, [float_type]
+tanh = 'tanh', float_type, [float_type]
+
+# power
+sqrt = 'sqrt', float_type, [float_type]
+pow = 'pow', float_type, [float_type, float_type]
+
+# exponential
+exp = 'exp', float_type, [float_type]
+log = 'log', float_type, [float_type]
+log10 = 'log10', float_type, [float_type]
+
+# misc
+abs = 'abs', float_type, [float_type]
+ceil = 'ceil', float_type, [float_type]
+floor = 'floor', float_type, [float_type]
+fmod = 'fmod', float_type, [float_type, float_type]
+
+#------------------------------------------------------------------------
+# Name Intrinsic Mapping
+#------------------------------------------------------------------------
+
+llvm_intrinsics = {
+ 'sin' : lc.INTR_SIN,
+ 'cos' : lc.INTR_COS,
+
+ 'sqrt' : lc.INTR_SQRT,
+ 'pow' : lc.INTR_POW,
+ 'exp' : lc.INTR_EXP,
+ 'log' : lc.INTR_LOG,
+ 'sqrt' : lc.INTR_SQRT,
+ 'abs' : lc.INTR_FABS,
+}
View
2  blaze/blir/opcodes.py
@@ -16,9 +16,9 @@ def def_op(name, op):
# BINARY_DIVIDE
# BINARY_MULTIPLY
# BINARY_SUBTRACT
-# CALL_FOREIGN
# CALL_FUNCTION
# COMPARE
+# DEF_FOREIGN
# GLOBAL
# LOAD
# LOAD_ARGUMENT
View
28 blaze/blir/passes.py
@@ -11,7 +11,6 @@
import exc
from threading import Lock
-from plyhacks import lexfrom, yaccfrom
compilelock = Lock()
@@ -117,12 +116,17 @@ def optimizer_pass(ast, env):
cgen = env['cgen']
lfunctions = env['lfunctions']
- optimizer = codegen.LLVMOptimizer(cgen.module)
+ opt_level = env['args']['O']
+ optimizer = codegen.LLVMOptimizer(cgen.module, opt_level)
+ # function-level optimize
for lfunc in lfunctions:
optimizer.run(lfunc)
lfunc.verify()
+ # module-level optimization
+ optimizer.runmodule(cgen.module)
+
cgen.module.verify()
env['lmodule'] = cgen.module
@@ -157,9 +161,11 @@ def linker_pass(ast, env):
# Toplevel
#------------------------------------------------------------------------
-def compile(source):
+def compile(source, **opts):
+ opts.setdefault('O', 2)
+ env = {'args': opts}
with compilelock:
- ast, env = compiler(source, {})
+ ast, env = compiler(source, env)
return ast, env
#------------------------------------------------------------------------
@@ -169,7 +175,8 @@ def compile(source):
def main():
import argparse
argp = argparse.ArgumentParser('blirc')
- argp.add_argument('file', metavar="file", nargs='?', help='Module')
+ argp.add_argument('file', metavar="file", nargs='?', help='Source file')
+ argp.add_argument('-O', metavar="opt", nargs='?', type=int, help='Optimization level', default=2)
argp.add_argument('--ddump-parse', action='store_true', help='Dump parse tree')
argp.add_argument('--ddump-lex', action='store_true', help='Dump token stream')
argp.add_argument('--ddump-blocks', action='store_true', help='Dump the block structure')
@@ -185,7 +192,7 @@ def main():
if args.file:
source = open(args.file).read()
else:
- print 'No input'
+ sys.stderr.write('No input\n')
sys.exit(1)
if args.ddump_lex:
@@ -207,7 +214,8 @@ def main():
# =====================================
start = time.time()
with errors.listen():
- ast, env = compile(source)
+ opts = vars(args)
+ ast, env = compile(source, **opts)
timing = time.time() - start
# =====================================
@@ -216,13 +224,15 @@ def main():
elif args.emit_x86:
print env['lmodule'].to_native_assembly()
elif args.run:
- exc.execute(env)
+ ctx = exc.Context(env)
+ exc.execute(ctx, fname='main')
else:
print 'Compile time %.3fs' % timing
except CompileError as e:
- print 'FAIL: Failure in compiler phase:', e.args[0]
+ sys.stderr.write('FAIL: Failure in compiler phase: %s\n' % e.args[0])
sys.exit(1)
+ errors.reset()
if __name__ == '__main__':
main()
View
26 blaze/blir/prelude.c
@@ -2,10 +2,6 @@
#include <string.h>
#include <stdlib.h>
-// Later...
-//#include <blosc.h>
-//#include <zmq.h>
-
//#include <mkl.h>
//#include <mkl_types.h>
//#include <mkl_lapack.h>
@@ -18,6 +14,14 @@ extern "C" {
#include "datashape.h"
// ------------------------------------------------------------------------
+
+typedef struct {
+ char *data;
+ int nd;
+ int *strides;
+} ndarray;
+
+// ------------------------------------------------------------------------
// Execution State
// ------------------------------------------------------------------------
@@ -43,16 +47,14 @@ void Blir_Finalize()
// Datashape Operations
// ------------------------------------------------------------------------
-// XXX Very limited for now...
-
-void is_fixed(type_t *ds)
+int is_fixed(type_t *ds)
{
- kindof(ds) == FIXED;
+ return kindof(ds) == FIXED;
}
-void is_ctype(type_t *ds)
+int is_ctype(type_t *ds)
{
- kindof(ds) == CTYPE;
+ return kindof(ds) == CTYPE;
}
// ------------------------------------------------------------------------
@@ -110,6 +112,10 @@ void show_bool(int b)
}
}
+void show_array(ndarray *a) {
+ printf("array(%p)", a->data);
+}
+
#ifdef __cplusplus
}
#endif
View
17 blaze/blir/typecheck.py
@@ -5,6 +5,7 @@
"""
import btypes
+import intrinsics
from syntax import *
from errors import error
@@ -107,6 +108,15 @@ def visit_BinOp(self, node):
node.type = node.left.type
def visit_FunctionCall(self, node):
+ if node.name in intrinsics.llvm_intrinsics:
+ for n in node.arglist:
+ self.visit(n)
+
+ # just pop the last type off the signature
+ retty = getattr(intrinsics, node.name)[2][-1]
+ node.type = retty
+ return
+
symnode = self.symtab.lookup(node.name, self.scope)
if symnode:
@@ -119,6 +129,7 @@ def visit_FunctionCall(self, node):
for n, (arg, parm) in enumerate(zip(node.arglist, symnode.parameters),1):
self.visit(arg)
+ arg.type == parm.type
if arg.type != parm.type:
error(arg.lineno,"Type Error. Argument %d must be %s" % (n, parm.type.name))
@@ -162,7 +173,7 @@ def visit_VarDecl(self, node):
if node.expr:
self.visit(node.expr)
if node.expr.type != node.type:
- error(node.lineno,"Type error %s = %s" % (node.type.name, node.expr.type.name))
+ error(node.lineno,"Type Error %s != %s" % (node.type.name, node.expr.type.name))
else:
# Possibly controversial decision:
# --------------------------------
@@ -260,7 +271,7 @@ def visit_LoadIndex(self, node):
# <array[int]> -> int
node.type = sym.type.arg.type
else:
- error(node.lineno,"Type Error: Cannot index scalar type" % node.name)
+ error(node.lineno,"Type Error: Cannot index scalar type: %s" % node.name)
node.type = btypes.undefined
else:
error(node.lineno,"Name Error: %s undefined" % node.name)
@@ -275,7 +286,7 @@ def visit_StoreIndex(self, node):
# extract the element pointer
node.type = sym.type.arg.type
else:
- error(node.lineno,"Type Error: Cannot index scalar type" % node.name)
+ error(node.lineno,"Type Error: Cannot index scalar type: %s" % node.name)
node.type = btypes.undefined
else:
error(node.lineno,"Name Error: %s undefined" % node.name)
View
52 blaze/carray/carrayExtension.pyx
@@ -423,17 +423,11 @@ cdef class chunk:
@property
def pointer(self):
- if self.memory:
- return <Py_uintptr_t> self.data+BLOSCPACK_HEADER_LENGTH
- else:
- raise RuntimeError("Not in memory")
+ return <Py_uintptr_t> self.data+BLOSCPACK_HEADER_LENGTH
@property
def viewof(self):
- if self.memory:
- return PyBuffer_FromMemory(<void*>self.data, <Py_ssize_t>self.cdbytes)
- else:
- raise RuntimeError("Not in memory")
+ return PyBuffer_FromMemory(<void*>self.data, <Py_ssize_t>self.cdbytes)
def __setitem__(self, object key, object value):
@@ -782,13 +776,13 @@ cdef class carray:
property nchunks:
def __get__(self):
# TODO: do we need to handle the last chunk specially?
- return cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ return <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
property partitions:
def __get__(self):
# Return a sequence of tuples indicating the bounds
# of each of the chunks.
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
chunklen = cython.cdiv(self._chunksize, self.atomsize)
return [(i*chunklen,(i+1)*chunklen) for i in xrange(nchunks)]
@@ -833,7 +827,7 @@ cdef class carray:
return len(self.chunks)
else:
# Important to do the cast in order to get a npy_intp result
- return cython.cdiv(self._nbytes, <npy_intp>self.atomsize)
+ return <npy_intp>cython.cdiv(self._nbytes, self.atomsize)
property mode:
"The mode used to create/open the `mode`."
@@ -1101,7 +1095,7 @@ cdef class carray:
# Compress data in chunks
cbytes = 0
chunklen = self._chunklen
- nchunks = cython.cdiv(nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(nbytes, self._chunksize)
for i from 0 <= i < nchunks:
assert i*chunklen < array_.size, "i, nchunks: %d, %d" % (i, nchunks)
chunk_ = chunk(array_[i*chunklen:(i+1)*chunklen],
@@ -1268,7 +1262,7 @@ cdef class carray:
# Then fill other possible chunks
nbytes = bsize - nbytesfirst
- nchunks = cython.cdiv(nbytes, <npy_intp>chunksize)
+ nchunks = <npy_intp>cython.cdiv(nbytes, chunksize)
chunklen = self._chunklen
# Get a new view skipping the elements that have been already copied
remainder = arrcpy[cython.cdiv(nbytesfirst, atomsize):]
@@ -1339,7 +1333,7 @@ cdef class carray:
leftover = leftover2 * atomsize
# Remove complete chunks
- nchunk2 = lnchunk = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunk2 = lnchunk = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
while nchunk2 > nchunk:
chunk_ = chunks.pop()
cbytes += chunk_.cbytes
@@ -1555,7 +1549,7 @@ cdef class carray:
# Get a container for the result
result = np.zeros(1, dtype=dtype)[0]
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
for nchunk from 0 <= nchunk < nchunks:
chunk_ = self.chunks[nchunk]
if chunk_.isconstant:
@@ -1595,9 +1589,9 @@ cdef class carray:
cdef chunk chunk_
atomsize = self.atomsize
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
chunklen = self._chunklen
- nchunk = cython.cdiv(pos, <npy_intp>chunklen)
+ nchunk = <npy_intp>cython.cdiv(pos, chunklen)
# Check whether pos is in the last chunk
if nchunk == nchunks and self.leftover:
@@ -1608,7 +1602,7 @@ cdef class carray:
# Locate the *block* inside the chunk
chunk_ = self.chunks[nchunk]
blocksize = chunk_.blocksize
- blocklen = cython.cdiv(blocksize, atomsize)
+ blocklen = <npy_intp>cython.cdiv(blocksize, atomsize)
if atomsize > blocksize:
# This request cannot be resolved here
@@ -1624,7 +1618,7 @@ cdef class carray:
# self._cbytes += chunksize
# Check if block is cached
- idxcache = cython.cdiv(pos, <npy_intp>blocklen) * blocklen
+ idxcache = <npy_intp>cython.cdiv(pos, blocklen) * blocklen
if idxcache == self.idxcache:
# Hit!
posinbytes = (pos % blocklen) * atomsize
@@ -1783,7 +1777,7 @@ cdef class carray:
# Fill it from data in chunks
nwrow = 0
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
if self.leftover > 0:
nchunks += 1
for nchunk from 0 <= nchunk < nchunks:
@@ -1924,7 +1918,7 @@ cdef class carray:
# Fill it from data in chunks
nwrow = 0
chunklen = self._chunklen
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
if self.leftover > 0:
nchunks += 1
for nchunk from 0 <= nchunk < nchunks:
@@ -1963,17 +1957,17 @@ cdef class carray:
cdef chunk chunk_
# Check that we are inside limits
- nrows = cython.cdiv(self._nbytes, <npy_intp>self.atomsize)
+ nrows = <npy_intp>cython.cdiv(self._nbytes, self.atomsize)
if (start + blen) > nrows:
blen = nrows - start
# Fill `out` from data in chunks
nwrow = 0
stop = start + blen
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
chunklen = cython.cdiv(self._chunksize, self.atomsize)
- schunk = cython.cdiv(start, <npy_intp>chunklen)
- echunk = cython.cdiv((start+blen), <npy_intp>chunklen)
+ schunk = <npy_intp>cython.cdiv(start, chunklen)
+ echunk = <npy_intp>cython.cdiv((start+blen), chunklen)
for nchunk from schunk <= nchunk <= echunk:
# Compute start & stop for each block
startb = start % chunklen
@@ -2009,10 +2003,10 @@ cdef class carray:
# Fill it from data in chunks
nwrow = 0
chunklen = self._chunklen
- nchunks = cython.cdiv(self._nbytes, <npy_intp>self._chunksize)
+ nchunks = <npy_intp>cython.cdiv(self._nbytes, self._chunksize)
if self.leftover > 0:
nchunks += 1
- nrows = cython.cdiv(self._nbytes, <npy_intp>self.atomsize)
+ nrows = <npy_intp>cython.cdiv(self._nbytes, self.atomsize)
for nchunk from 0 <= nchunk < nchunks:
# Compute start & stop for each block
startb, stopb, _ = clip_chunk(nchunk, chunklen, 0, nrows, 1)
@@ -2048,7 +2042,7 @@ cdef class carray:
if not self.sss_mode:
self.start = 0
- self.stop = cython.cdiv(self._nbytes, <npy_intp>self.atomsize)
+ self.stop = <npy_intp>cython.cdiv(self._nbytes, self.atomsize)
self.step = 1
if not (self.sss_mode or self.where_mode or self.wheretrue_mode):
self.nhits = 0
@@ -2295,7 +2289,7 @@ cdef class carray:
if isinstance(barr, carray):
# Check for zero'ed chunks in carrays
carr = barr
- nchunk = cython.cdiv(self.nrowsread, <npy_intp>self.nrowsinbuf)
+ nchunk = <npy_intp>cython.cdiv(self.nrowsread, self.nrowsinbuf)
if nchunk < len(carr.chunks):
chunk_ = carr.chunks[nchunk]
if chunk_.isconstant and chunk_.constant in (0, ''):
View
6 blaze/carray/tests/test_carray.py
@@ -1234,8 +1234,10 @@ def test07(self):
"""
types = [np.int8, np.int16, np.int32, np.int64,
np.uint8, np.uint16, np.uint32, np.uint64,
- np.float16, np.float32, np.float64, np.float128,
- np.complex64, np.complex128, np.complex256]
+ np.float16, np.float32, np.float64,
+ np.complex64, np.complex128]
+ if hasattr(np, 'float128'):
+ types.extend([np.float128, np.complex256])
shapes = [(10,), (10,10), (10,10,10)]
for shape in shapes:
for t in types:
View
38 blaze/carray/tests/test_large_carray.py
@@ -19,59 +19,59 @@ def test00(self):
"""Creating an extremely large carray (> 2**32) in memory."""
cn = ca.zeros(5e9, dtype="i1")
- self.assert_(len(cn) == int(5e9))
+ self.assertEqual(len(cn), int(5e9))
# Now check some accesses
cn[1] = 1
- self.assert_(cn[1] == 1)
+ self.assertEqual(cn[1], 1)
cn[int(2e9)] = 2
- self.assert_(cn[int(2e9)] == 2)
+ self.assertEqual(cn[int(2e9)], 2)
cn[long(3e9)] = 3
- self.assert_(cn[long(3e9)] == 3)
+ self.assertEqual(cn[long(3e9)], 3)
cn[-1] = 4
- self.assert_(cn[-1] == 4)
+ self.assertEqual(cn[-1], 4)
- self.assert_(cn.sum() == 10)
+ self.assertEqual(cn.sum(), 10)
def test01(self):
"""Creating an extremely large carray (> 2**32) on disk."""
cn = ca.zeros(5e9, dtype="i1", rootdir=self.rootdir)
- self.assert_(len(cn) == int(5e9))
+ self.assertEqual(len(cn), int(5e9))
# Now check some accesses
cn[1] = 1
- self.assert_(cn[1] == 1)
+ self.assertEqual(cn[1], 1)
cn[int(2e9)] = 2
- self.assert_(cn[int(2e9)] == 2)
+ self.assertEqual(cn[int(2e9)], 2)
cn[long(3e9)] = 3
- self.assert_(cn[long(3e9)] == 3)
+ self.assertEqual(cn[long(3e9)], 3)
cn[-1] = 4
- self.assert_(cn[-1] == 4)
+ self.assertEqual(cn[-1], 4)
- self.assert_(cn.sum() == 10)
+ self.assertEqual(cn.sum(), 10)
def test02(self):
"""Opening an extremely large carray (> 2**32) on disk."""
# Create the array on-disk
cn = ca.zeros(5e9, dtype="i1", rootdir=self.rootdir)
- self.assert_(len(cn) == int(5e9))
+ self.assertEqual(len(cn), int(5e9))
# Reopen it from disk
cn = ca.carray(rootdir=self.rootdir)
- self.assert_(len(cn) == int(5e9))
+ self.assertEqual(len(cn), int(5e9))
# Now check some accesses
cn[1] = 1
- self.assert_(cn[1] == 1)
+ self.assertEqual(cn[1], 1)
cn[int(2e9)] = 2
- self.assert_(cn[int(2e9)] == 2)
+ self.assertEqual(cn[int(2e9)], 2)
cn[long(3e9)] = 3
- self.assert_(cn[long(3e9)] == 3)
+ self.assertEqual(cn[long(3e9)], 3)
cn[-1] = 4
- self.assert_(cn[-1] == 4)
+ self.assertEqual(cn[-1], 4)
- self.assert_(cn.sum() == 10)
+ self.assertEqual(cn.sum(), 10)
## Local Variables:
## mode: python
View
0  blaze/cgen/__init__.py
No changes.
View
572 blaze/cgen/astprint.py
@@ -0,0 +1,572 @@
+# -*- coding: utf-8 -*-
+"""
+ codegen
+ ~~~~~~~
+
+ Extension to ast that allow ast -> python code generation.
+
+ :copyright: Copyright 2008 by Armin Ronacher.
+ :license: BSD.
+"""
+from ast import *
+
+
+BOOLOP_SYMBOLS = {
+ And: 'and',
+ Or: 'or'
+}
+
+BINOP_SYMBOLS = {
+ Add: '+',
+ Sub: '-',
+ Mult: '*',
+ Div: '/',
+ FloorDiv: '//',
+ Mod: '%',
+ LShift: '<<',
+ RShift: '>>',
+ BitOr: '|',
+ BitAnd: '&',
+ BitXor: '^'
+}
+
+CMPOP_SYMBOLS = {
+ Eq: '==',
+ Gt: '>',
+ GtE: '>=',
+ In: 'in',
+ Is: 'is',
+ IsNot: 'is not',
+ Lt: '<',
+ LtE: '<=',
+ NotEq: '!=',
+ NotIn: 'not in'
+}
+
+UNARYOP_SYMBOLS = {
+ Invert: '~',
+ Not: 'not',
+ UAdd: '+',
+ USub: '-'
+}
+
+ALL_SYMBOLS = {}
+ALL_SYMBOLS.update(BOOLOP_SYMBOLS)
+ALL_SYMBOLS.update(BINOP_SYMBOLS)
+ALL_SYMBOLS.update(CMPOP_SYMBOLS)
+ALL_SYMBOLS.update(UNARYOP_SYMBOLS)
+
+
+def to_source(node, indent_with=' ' * 4, add_line_information=False):
+ """This function can convert a node tree back into python sourcecode.
+ This is useful for debugging purposes, especially if you're dealing with
+ custom asts not generated by python itself.
+
+ It could be that the sourcecode is evaluable when the AST itself is not
+ compilable / evaluable. The reason for this is that the AST contains some
+ more data than regular sourcecode does, which is dropped during
+ conversion.
+
+ Each level of indentation is replaced with `indent_with`. Per default this
+ parameter is equal to four spaces as suggested by PEP 8, but it might be
+ adjusted to match the application's styleguide.
+
+ If `add_line_information` is set to `True` comments for the line numbers
+ of the nodes are added to the output. This can be used to spot wrong line
+ number information of statement nodes.
+ """
+ generator = SourceGenerator(indent_with, add_line_information)
+ generator.visit(node)
+ return ''.join(generator.result)
+
+
+class SourceGenerator(NodeVisitor):
+ """This visitor is able to transform a well formed syntax tree into python
+ sourcecode. For more details have a look at the docstring of the
+ `node_to_source` function.
+ """
+
+ def __init__(self, indent_with, add_line_information=False):
+ self.result = []
+ self.indent_with = indent_with
+ self.add_line_information = add_line_information
+ self.indentation = 0
+ self.new_lines = 0
+
+ def write(self, x):
+ if self.new_lines:
+ if self.result:
+ self.result.append('\n' * self.new_lines)
+ self.result.append(self.indent_with * self.indentation)
+ self.new_lines = 0
+ self.result.append(x)
+
+ def newline(self, node=None, extra=0):
+ self.new_lines = max(self.new_lines, 1 + extra)
+ if node is not None and self.add_line_information:
+ self.write('# line: %s' % node.lineno)
+ self.new_lines = 1
+
+ def body(self, statements):
+ self.new_line = True
+ self.indentation += 1
+ for stmt in statements:
+ self.visit(stmt)
+ self.indentation -= 1
+
+ def body_or_else(self, node):
+ self.body(node.body)
+ if node.orelse:
+ self.newline()
+ self.write('else:')
+ self.body(node.orelse)
+
+ def signature(self, node):
+ want_comma = []
+ def write_comma():
+ if want_comma:
+ self.write(', ')
+ else:
+ want_comma.append(True)
+
+ padding = [None] * (len(node.args) - len(node.defaults))
+ for arg, default in zip(node.args, padding + node.defaults):
+ write_comma()
+ self.visit(arg)
+ if default is not None:
+ self.write('=')
+ self.visit(default)
+ if node.vararg is not None:
+ write_comma()
+ self.write('*' + node.vararg)
+ if node.kwarg is not None:
+ write_comma()
+ self.write('**' + node.kwarg)
+
+ def decorators(self, node):
+ for decorator in node.decorator_list:
+ self.newline(decorator)
+ self.write('@')
+ self.visit(decorator)
+
+ # Statements
+
+ def visit_Assign(self, node):
+ self.newline(node)
+ for idx, target in enumerate(node.targets):
+ if idx:
+ self.write(', ')
+ self.visit(target)
+ self.write(' = ')
+ self.visit(node.value)
+
+ def visit_AugAssign(self, node):
+ self.newline(node)
+ self.visit(node.target)
+ self.write(BINOP_SYMBOLS[type(node.op)] + '=')
+ self.visit(node.value)
+
+ def visit_ImportFrom(self, node):
+ self.newline(node)
+ self.write('from %s%s import ' % ('.' * node.level, node.module))
+ for idx, item in enumerate(node.names):
+ if idx:
+ self.write(', ')
+ self.write(item)
+
+ def visit_Import(self, node):
+ self.newline(node)
+ for item in node.names:
+ self.write('import ')
+ self.visit(item)
+
+ def visit_Expr(self, node):
+ self.newline(node)
+ self.generic_visit(node)
+
+ def visit_FunctionDef(self, node):
+ self.newline(extra=1)
+ self.decorators(node)
+ self.newline(node)
+ self.write('def %s(' % node.name)
+ self.signature(node.args)
+ self.write('):')
+ self.body(node.body)
+
+ def visit_ClassDef(self, node):
+ have_args = []
+ def paren_or_comma():
+ if have_args:
+ self.write(', ')
+ else:
+ have_args.append(True)
+ self.write('(')
+
+ self.newline(extra=2)
+ self.decorators(node)
+ self.newline(node)
+ self.write('class %s' % node.name)
+ for base in node.bases:
+ paren_or_comma()
+ self.visit(base)
+ # XXX: the if here is used to keep this module compatible
+ # with python 2.6.
+ if hasattr(node, 'keywords'):
+ for keyword in node.keywords:
+ paren_or_comma()
+ self.write(keyword.arg + '=')
+ self.visit(keyword.value)
+ if node.starargs is not None:
+ paren_or_comma()
+ self.write('*')
+ self.visit(node.starargs)
+ if node.kwargs is not None:
+ paren_or_comma()
+ self.write('**')
+ self.visit(node.kwargs)
+ self.write(have_args and '):' or ':')
+ self.body(node.body)
+
+ def visit_If(self, node):
+ self.newline(node)
+ self.write('if ')
+ self.visit(node.test)
+ self.write(':')
+ self.body(node.body)
+ while True:
+ else_ = node.orelse
+ if len(else_) == 1 and isinstance(else_[0], If):
+ node = else_[0]
+ self.newline()
+ self.write('elif ')
+ self.visit(node.test)
+ self.write(':')
+ self.body(node.body)
+ else:
+ self.newline()
+ self.write('else:')
+ self.body(else_)
+ break
+
+ def visit_For(self, node):
+ self.newline(node)
+ self.write('for ')
+ self.visit(node.target)
+ self.write(' in ')
+ self.visit(node.iter)
+ self.write(':')
+ self.body_or_else(node)
+
+ def visit_While(self, node):
+ self.newline(node)
+ self.write('while ')
+ self.visit(node.test)
+ self.write(':')
+ self.body_or_else(node)
+
+ def visit_With(self, node):
+ self.newline(node)
+ self.write('with ')
+ self.visit(node.context_expr)
+ if node.optional_vars is not None:
+ self.write(' as ')
+ self.visit(node.optional_vars)
+ self.write(':')
+ self.body(node.body)
+
+ def visit_Pass(self, node):
+ self.newline(node)
+ self.write('pass')
+
+ def visit_Print(self, node):
+ # XXX: python 2.6 only
+ self.newline(node)
+ self.write('print ')
+ want_comma = False
+ if node.dest is not None:
+ self.write(' >> ')
+ self.visit(node.dest)
+ want_comma = True
+ for value in node.values:
+ if want_comma:
+ self.write(', ')
+ self.visit(value)
+ want_comma = True
+ if not node.nl:
+ self.write(',')
+
+ def visit_Delete(self, node):
+ self.newline(node)
+ self.write('del ')
+ for idx, target in enumerate(node):
+ if idx:
+ self.write(', ')
+ self.visit(target)
+
+ def visit_TryExcept(self, node):
+ self.newline(node)
+ self.write('try:')
+ self.body(node.body)
+ for handler in node.handlers:
+ self.visit(handler)
+
+ def visit_TryFinally(self, node):
+ self.newline(node)
+ self.write('try:')
+ self.body(node.body)
+ self.newline(node)
+ self.write('finally:')
+ self.body(node.finalbody)
+
+ def visit_Global(self, node):
+ self.newline(node)
+ self.write('global ' + ', '.join(node.names))
+
+ def visit_Nonlocal(self, node):
+ self.newline(node)
+ self.write('nonlocal ' + ', '.join(node.names))
+
+ def visit_Return(self, node):
+ self.newline(node)
+ self.write('return ')
+ self.visit(node.value)
+
+ def visit_Break(self, node):
+ self.newline(node)
+ self.write('break')
+
+ def visit_Continue(self, node):
+ self.newline(node)
+ self.write('continue')
+
+ def visit_Raise(self, node):
+ # XXX: Python 2.6 / 3.0 compatibility
+ self.newline(node)
+ self.write('raise')
+ if hasattr(node, 'exc') and node.exc is not None:
+ self.write(' ')
+ self.visit(node.exc)
+ if node.cause is not None:
+ self.write(' from ')
+ self.visit(node.cause)
+ elif hasattr(node, 'type') and node.type is not None:
+ self.visit(node.type)
+ if node.inst is not None:
+ self.write(', ')
+ self.visit(node.inst)
+ if node.tback is not None:
+ self.write(', ')
+ self.visit(node.tback)
+
+ # Expressions
+
+ def visit_Attribute(self, node):
+ self.visit(node.value)
+ self.write('.' + node.attr)
+
+ def visit_Call(self, node):
+ want_comma = []
+ def write_comma():
+ if want_comma:
+ self.write(', ')
+ else:
+ want_comma.append(True)
+
+ self.visit(node.func)
+ self.write('(')
+ for arg in node.args:
+ write_comma()
+ self.visit(arg)
+ for keyword in node.keywords:
+ write_comma()
+ self.write(keyword.arg + '=')
+ self.visit(keyword.value)
+ if node.starargs is not None:
+ write_comma()
+ self.write('*')
+ self.visit(node.starargs)
+ if node.kwargs is not None:
+ write_comma()
+ self.write('**')
+ self.visit(node.kwargs)
+ self.write(')')
+
+ def visit_Name(self, node):
+ self.write(node.id)
+
+ def visit_Str(self, node):
+ self.write(repr(node.s))
+
+ def visit_Bytes(self, node):
+ self.write(repr(node.s))
+
+ def visit_Num(self, node):
+ self.write(repr(node.n))
+
+ def visit_Tuple(self, node):
+ self.write('(')
+ idx = -1
+ for idx, item in enumerate(node.elts):
+ if idx: