Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Rearranged OpenCL code

The OpenCL module has been rearranged to show its focus on replacing the
inverse distance trees.  This will allow me to add more OpenCL modules
for other parts of the code.

idt.cl: renamed from nearest.cl
nearest.cl: renamed to idt.cl
newcl.py: renamed to newclidt.py
newclidt.py: renamed from newcl.py
newcrust.py: using new names
newregion.py: using new names
  • Loading branch information...
commit 16db65e6314101b1c95eb654b90b06390475215c 1 parent e9c4b73
@mathuin authored
View
0  nearest.cl → idt.cl
File renamed without changes
View
95 newcl.py → newclidt.py
@@ -1,6 +1,5 @@
-# new CL module
-from numpy import asarray, array, int32, zeros, empty_like
-import numpy
+# new CL IDT module
+import numpy as n
from timer import timer
from itertools import product
from random import randint, uniform
@@ -8,16 +7,17 @@
from multiprocessing import Pool
try:
import pyopencl as cl
+ import pyopencl.array as cla
except ImportError:
pass
-class CL:
+class CLIDT:
"""Use OpenCL or Invdisttree to solve the IDT problem."""
- # value at which tests show performance decreases as arrays grow
- IDTmaxsize = 2048
# value at which video card refuses to run while X is on
OpenCLmaxsize = 512
+ # largest amount of indices which can be processed at once
+ indexmaxsize = 8 * 1024 * 1024
# default value for nearest neighbors
nnear = 11
@@ -27,30 +27,29 @@ def genindices(self, arrayin):
"""Generate indices for splitting array."""
retval = dict()
# run the 'trim' program
- template = empty_like(arrayin)
- arrayin_buf = cl.Buffer(self.ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=arrayin)
- template_buf = cl.Buffer(self.ctx, cl.mem_flags.WRITE_ONLY, size=template.nbytes)
- self.program.trim(self.queue, arrayin.shape, None, arrayin_buf, template_buf, int32(self.split))
- arrayout = empty_like(arrayin)
- try:
- cl.enqueue_copy(self.queue, arrayout, template_buf)
- except AttributeError:
- cl.enqueue_read_buffer(self.queue, template_buf, arrayout).wait()
- # splitting is harder than I thought.
- for index, elem in enumerate(arrayout):
- splitkey = tuple([x for x in elem],)
- try:
- retval[splitkey]
- except KeyError:
- retval[splitkey] = []
- retval[splitkey].append(index)
+ # need to split if it's too long!
+ splitlist = tuple([x for x in xrange(CLIDT.indexmaxsize, arrayin.shape[0], CLIDT.indexmaxsize)])
+ indexinc = 0
+ for chunk in n.vsplit(arrayin, splitlist):
+ chunkarr = cla.to_device(self.queue, n.asarray(chunk, dtype=n.int32))
+ template = cla.empty_like(chunkarr)
+ event = self.program.trim(self.queue, chunkarr.shape, None, chunkarr.data, template.data, n.int32(self.split))
+ event.wait()
+ for index, elem in enumerate(template.get()):
+ splitkey = tuple([x for x in elem],)
+ try:
+ retval[splitkey]
+ except KeyError:
+ retval[splitkey] = []
+ retval[splitkey].append(index+indexinc)
+ indexinc += CLIDT.indexmaxsize
return retval
@timer()
def __init__(self, coords, values, base, wantCL=True, split=None, nnear=None, majority=True):
- self.coords = asarray(coords, dtype=int32)
- self.values = asarray(values, dtype=int32)
- self.base = asarray(base, dtype=int32)
+ self.coords = n.asarray(coords, dtype=n.int32)
+ self.values = n.asarray(values, dtype=n.int32)
+ self.base = n.asarray(base, dtype=n.int32)
(lencoords, null) = self.coords.shape
(lenvalues,) = self.values.shape
(lenbase, null) = self.base.shape
@@ -59,13 +58,15 @@ def __init__(self, coords, values, base, wantCL=True, split=None, nnear=None, ma
self.wantCL = wantCL
if self.wantCL == True:
if split == None:
- self.split = CL.OpenCLmaxsize
+ self.split = CLIDT.OpenCLmaxsize
else:
self.split = split
try:
+ import pyopencl as cl
+ import pyopencl.array as cla
self.ctx = cl.create_some_context()
self.queue = cl.CommandQueue(self.ctx)
- filestr = ''.join(open('nearest.cl', 'r').readlines())
+ filestr = ''.join(open('idt.cl', 'r').readlines())
self.program = cl.Program(self.ctx, filestr).build()
self.coordindices = self.genindices(self.coords)
self.baseindices = self.genindices(self.base)
@@ -76,38 +77,30 @@ def __init__(self, coords, values, base, wantCL=True, split=None, nnear=None, ma
self.canCL = False
if nnear == None:
- self.nnear = CL.nnear
+ self.nnear = n.int32(CLIDT.nnear)
else:
- self.nnear = nnear
+ self.nnear = n.int32(nnear)
- if majority == True:
- self.usemajority = 1
- else:
- self.usemajority = 0
+ self.usemajority = n.int32(1 if majority else 0)
def build(self, coords, values, base):
(lenbase, null) = base.shape
(lencoords, null) = coords.shape
- template = zeros((lenbase), dtype=int32)
- coords_buf = cl.Buffer(self.ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=coords)
- values_buf = cl.Buffer(self.ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=values)
- base_buf = cl.Buffer(self.ctx, cl.mem_flags.READ_ONLY | cl.mem_flags.COPY_HOST_PTR, hostbuf=base)
- template_buf = cl.Buffer(self.ctx, cl.mem_flags.WRITE_ONLY, size=template.nbytes)
- self.program.nearest(self.queue, base.shape, None, coords_buf, values_buf, base_buf, template_buf, int32(lencoords), int32(self.nnear), int32(self.usemajority))
- suboutput = empty_like(template)
- try:
- cl.enqueue_copy(self.queue, suboutput, template_buf)
- except AttributeError:
- cl.enqueue_read_buffer(self.queue, template_buf, suboutput).wait()
+ coords_array = cla.to_device(self.queue, coords)
+ values_array = cla.to_device(self.queue, values)
+ base_array = cla.to_device(self.queue, base)
+ template_array = cla.zeros(self.queue, (lenbase), dtype=n.int32)
+ event = self.program.nearest(self.queue, base.shape, None, coords_array.data, values_array.data, base_array.data, template_array.data, n.int32(lencoords), self.nnear, self.usemajority)
+ event.wait()
- return suboutput
+ return template_array.get()
@timer()
def __call__(self):
# build output array
if self.wantCL and self.canCL:
(lenbase, null) = self.base.shape
- retval = zeros((lenbase), dtype=int32)
+ retval = n.zeros((lenbase), dtype=n.int32)
for key, value in self.baseindices.items():
(a, b) = key
cindices = []
@@ -121,7 +114,7 @@ def __call__(self):
retval[value] = self.build(coords, values, base)
else:
IDT = Invdisttree(self.coords, self.values)
- retval = asarray(IDT(self.base, self.nnear, majority=(self.usemajority==1)), dtype=int32)
+ retval = n.asarray(IDT(self.base, self.nnear, majority=(self.usemajority==1)), dtype=n.int32)
return retval
def __del__(self):
@@ -139,9 +132,9 @@ def __del__(self):
coverage = 0.05
numcoords = int(xsize*zsize*coverage)
shape = (zsize, xsize)
- base = array([(z, x) for z, x in product(xrange(zsize), xrange(xsize))], dtype=int32)
- coords = array([(randint(0, zsize-1), randint(0, xsize-1)) for elem in xrange(numcoords)], dtype=int32)
- values = array([uniform(CL.minwidth, CL.maxwidth) for elem in xrange(numcoords)], dtype=int32)
+ base = n.array([(z, x) for z, x in product(xrange(zsize), xrange(xsize))], dtype=n.int32)
+ coords = n.array([(randint(0, zsize-1), randint(0, xsize-1)) for elem in xrange(numcoords)], dtype=n.int32)
+ values = n.array([uniform(1, 5) for elem in xrange(numcoords)], dtype=n.int32)
print 'initializing CL object of size %d, %d with (forced) splitting...' % (xsize, zsize)
yes = CL(coords, values, base, split=xsize)
View
6 newcrust.py
@@ -1,7 +1,7 @@
# new crust module
from itertools import product
from random import randint, uniform
-from newcl import CL
+from newclidt import CLIDT
class Crust:
"""Smoothly irregular crust between the surface and the underlying stone."""
@@ -20,10 +20,10 @@ def __init__(self, xsize, zsize, coverage=0.05, wantCL=True):
self.coords = [(randint(0, self.zsize-1), randint(0, self.xsize-1)) for elem in xrange(self.numcoords)]
self.values = [uniform(Crust.minwidth, Crust.maxwidth) for elem in xrange(self.numcoords)]
self.base = [(z, x) for z, x in product(xrange(self.zsize), xrange(self.xsize))]
- self.cl = CL(self.coords, self.values, self.base, wantCL=self.wantCL, majority=False)
+ self.clidt = CLIDT(self.coords, self.values, self.base, wantCL=self.wantCL, majority=False)
def __call__(self):
- retval = self.cl()
+ retval = self.clidt()
retval.resize((self.zsize, self.xsize))
return retval
View
8 newregion.py
@@ -33,7 +33,7 @@
from itertools import product
from random import uniform, randint
#
-from newcl import CL
+from newclidt import CLIDT
class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
"""stupid redirect handling craziness"""
@@ -626,14 +626,14 @@ def buildmap(self):
depthyrange = [lcextents['ymax'] - self.scale * y for y in xrange(depthylen)]
depthbase = numpy.array([(x, y) for y in depthyrange for x in depthxrange])
# 4. an inverse distance tree must be built from that
- lcCL = CL(coords, values, depthbase, wantCL=True)
+ lcCLIDT = CLIDT(coords, values, depthbase, wantCL=True)
#lcIDT = Invdisttree(coords, values)
# 5. the desired output comes from that inverse distance tree
#deptharray = lcIDT(depthbase, nnear=11, majority=True)
- deptharray = lcCL()
+ deptharray = lcCLIDT()
deptharray.resize((depthylen, depthxlen))
#lcIDT = None
- lcCL = None
+ lcCLIDT = None
print "new code finished in %.2f seconds." % (clock()-initial)
else:
warpcmd = 'rm -rf %s && gdalwarp -q -multi -t_srs "%s" -tr %d %d -te %d %d %d %d -r near %s %s' % (lcfile, Region.t_srs, self.scale, self.scale, lcextents['xmin'], lcextents['ymin'], lcextents['xmax'], lcextents['ymax'], lcvrt, lcfile)
Please sign in to comment.
Something went wrong with that request. Please try again.