adding explicit numpy.dtype calls to avoid numpy bug

nbodykit/core/algorithms/FOF6D.py

@@ -47,12 +47,13 @@ def run(self):
         with self.halolabel.open() as stream:
             [[Label]] = stream.read(['Label'], full=True)
         mask = Label != 0
-        PIG = numpy.empty(mask.sum(), dtype=[
+        dtype = numpy.dtype([
                 ('Position', ('f4', 3)), 
                 ('Velocity', ('f4', 3)), 
                 ('Label', ('i4')), 
                 ('Rank', ('i4')), 
                 ])
+        PIG = numpy.empty(mask.sum(), dtype=dtype)
         PIG['Label'] = Label[mask]
         del Label
         with self.datasource.open() as stream:
@@ -150,7 +151,7 @@ def subfof(pos, vel, ll, vfactor, haloid, Ntot, boxsize):
         thresh *= 0.9
     # if nothing is found then assume this FOF group is a fluke.
 
-    output = numpy.empty(Nsub, dtype=[
+    dtype = numpy.dtype([
         ('Position', ('f4', 3)),
         ('Velocity', ('f4', 3)),
         ('LinkingLength', 'f4'),
@@ -162,6 +163,7 @@ def subfof(pos, vel, ll, vfactor, haloid, Ntot, boxsize):
         ('Length', 'i4'),
         ('HaloID', 'i4'),
         ])
+    output = numpy.empty(Nsub, dtype=dtype)
 
     output['Position'][...] = fof.center()[:Nsub, :3]
     output['Length'][...] = fof.length[:Nsub]

nbodykit/core/algorithms/FiberCollisions.py

@@ -105,7 +105,7 @@ def run(self):
 
         # return a structured array
         d = list(zip(['Label', 'Collided', 'NeighborID'], [labels, collided, neighbors]))
-        dtype = [(col, x.dtype) for col, x in d]
+        dtype = numpy.dtype([(col, x.dtype) for col, x in d])
         result = numpy.empty(len(labels), dtype=dtype)
         for col, x in d: result[col] = x
         return result
@@ -121,14 +121,15 @@ def _assign_fibers(self, Label):
 
         comm = self.comm
         mask = Label != 0
-        PIG = numpy.empty(mask.sum(), dtype=[
+        dtype = numpy.dtype([
                 ('Position', ('f4', 3)),  
                 ('Label', ('i4')), 
                 ('Rank', ('i4')), 
                 ('Index', ('i4')),
                 ('Collided', ('i4')),
                 ('NeighborID', ('i4'))
                 ])
+        PIG = numpy.empty(mask.sum(), dtype=dtype)
         PIG['Label'] = Label[mask]
         size = len(Label)
         size = comm.allgather(size)

nbodykit/core/algorithms/TraceHalo.py

@@ -40,11 +40,12 @@ def run(self):
         mpsort.sort(label, orderby=ID, comm=self.comm)
         del ID
 
-        data = numpy.empty(len(label), dtype=[
+        dtype = numpy.dtype([
                     ('ID', ('i8')), 
                     ('Position', ('f4', 3)), 
                     ('Velocity', ('f4', 3)), 
                     ])
+        data = numpy.empty(len(label), dtype=dtype)
         with self.dest.open() as dest:
             [[data['Position'][...]]] = dest.read(['Position'], full=True)
             [[data['Velocity'][...]]] = dest.read(['Velocity'], full=True)
@@ -71,12 +72,12 @@ def save(self, output, data):
             self.logger.info("N %s" % str(N))
             N[0] = 0
             with h5py.File(output, 'w') as ff:
-                data = numpy.empty(shape=(len(N),), 
-                    dtype=[
+
+                dtype = numpy.dtype([
                     ('Position', ('f4', 3)),
                     ('Velocity', ('f4', 3)),
                     ('Length', 'i4')])
-
+                data = numpy.empty(shape=(len(N),), dtype=dtype)
                 data['Position'] = hpos
                 data['Velocity'] = hvel
                 data['Length'] = N

nbodykit/core/datasource/FOFGroups.py

@@ -67,15 +67,15 @@ def readall(self):
         dataset = h5py.File(self.path, mode='r')[self.dataset]
         data = dataset[...]
 
-        data2 = numpy.empty(len(data),
-            dtype=[
+        dtype = numpy.dtype([
                 ('Position', ('f4', 3)),
                 ('Velocity', ('f4', 3)),
                 ('Mass', 'f4'),
                 ('Weight', 'f4'),
                 ('Length', 'i4'),
                 ('Rank', 'i4'),
                 ('LogMass', 'f4')])
+        data2 = numpy.empty(len(data),dtype=dtype)
 
         data2['Mass'] = data['Length'] * self.m0
         data2['Weight'] = 1.0

nbodykit/core/datasource/Pandas.py

@@ -129,7 +129,8 @@ def readall(self):
                 dtype[i] = (col, dt)
 
         # create the structured array from the DataFrame
-        toret = numpy.empty(len(data), dtype=dtype)
+        np_dtype = numpy.dtype(dtype)
+        toret = numpy.empty(len(data), dtype=np_dtype)
         for i, col in enumerate(data.columns):
             toret[col] = data[col].values.astype(dtype[i][1])
         toret = extend_dtype(toret, new_dtypes)

nbodykit/core/datasource/RaDecRedshift.py

@@ -146,6 +146,7 @@ def readall(self):
             dtype += [('Weight', 'f4')]
         if self.nbar_col is not None:
             dtype += [('Nbar', 'f4')]
+        dtype = numpy.dtype(dtype)
         new = numpy.empty(len(data), dtype=dtype)
 
         # rescale the angles

nbodykit/core/datasource/Subsample.py

@@ -54,11 +54,11 @@ def readall(self):
         dataset = h5py.File(self.path, mode='r')[self.dataset]
         data = dataset[...]
 
-        data2 = numpy.empty(len(data),
-            dtype=[
+        dtype = numpy.dtype([
                 ('Position', ('f4', 3)),
                 ('Velocity', ('f4', 3))
                 ])
+        data2 = numpy.empty(len(data),dtype=dtype)
 
         data2['Position'] = data['Position'] * self.BoxSize
         data2['Velocity'] = data['Velocity'] * self.BoxSize

nbodykit/dataset.py

@@ -177,8 +177,8 @@ def __init__(self, dims, edges, variables, **kwargs):
             self.coords[dim] = 0.5 * (edges[i][1:] + edges[i][:-1])
 
         # store variables as a structured array
-        dtypes = [(name, variables[name].dtype) for name in variables]            
-        self.data = numpy.empty(self.shape, dtype=numpy.dtype(dtypes))
+        dtypes = numpy.dtype([(name, variables[name].dtype) for name in variables])
+        self.data = numpy.empty(self.shape, dtype=dtypes)
         for name in variables:
             self.data[name] = variables[name]
 
@@ -304,6 +304,7 @@ def __setitem__(self, key, data):
         dtype += [(key, data.dtype.type)]
 
         # add old variables
+        dtype = numpy.dtype(dtype)
         new = numpy.zeros(self.data.shape, dtype=dtype)
         for col in names:
             new[col] = self.data[col]

nbodykit/distributedarray.py

@@ -115,7 +115,8 @@ def ScatterArray(data, comm, root=0):
 
     # initialize empty data on non-root ranks
     if comm.rank != root:
-        data = numpy.empty(0, dtype=(dtype, shape[1:]))
+        np_dtype = numpy.dtype((dtype, shape[1:]))
+        data = numpy.empty(0, dtype=np_dtype)
 
     # setup the custom dtype 
     duplicity = numpy.product(numpy.array(shape[1:], 'intp'))
@@ -366,8 +367,8 @@ def bincount(self, local=False):
 
 def test():
     comm = MPI.COMM_WORLD
-    local = numpy.empty((comm.rank), 
-            dtype=[('key', 'u8'), ('value', 'u8'), ('rank', 'i8')])
+    dtype = numpy.dtype([('key', 'u8'), ('value', 'u8'), ('rank', 'i8')])
+    local = numpy.empty((comm.rank), dtype=dtype)
     d = DistributedArray(local)
     local['key'] = numpy.arange(len(local))
     local['value'] = d.comm.rank * 10 + local['key']

nbodykit/fof.py

@@ -39,10 +39,11 @@ def fof_halo_label(minid, comm, thresh):
         particles that are in halos that contain less than thresh particles.
 
     """
-    data = numpy.empty(len(minid), dtype=[
+    dtype = numpy.dtype([
             ('origind', 'u8'), 
             ('fofid', 'u8'),
             ])
+    data = numpy.empty(len(minid), dtype=dtype)
     # assign origind for recovery of ordering, since
     # we need to work in sorted fofid 
     data['fofid'] = minid
@@ -242,8 +243,7 @@ def fof_catalogue(datasource, label, comm, calculate_initial=False):
             does not correspond to any halo.
 
     """
-    dtype=[
-        ('Position', ('f4', 3)),
+    dtype=[('Position', ('f4', 3)),
         ('Velocity', ('f4', 3)),
         ('Length', 'i4')]
 
@@ -279,6 +279,7 @@ def fof_catalogue(datasource, label, comm, calculate_initial=False):
     if comm.rank == 0: logger.info("Length = %s " % N[1:])
     if comm.rank == 0: logger.info("%d particles not in halo" % N[0])
 
+    dtype = numpy.dtype(dtype)
     if comm.rank == 0:
         catalogue = numpy.empty(shape=len(N), dtype=dtype)