Merge 36b42a4 into 5cdaa03

.travis.yml

@@ -4,8 +4,10 @@ language: python
 
 python:
     - 2.7
+    - 3.6
     # PyPy versions
     - pypy2.7-6.0
+    - pypy3
 
 services:
     - docker
@@ -28,6 +30,7 @@ jobs:
           python: 2.7
           script:
             - make docker-cpu
+            - make docker
             - bash ./.travis_deploy_docs.sh
 
 script:
@@ -39,7 +42,7 @@ script:
         simplejson \
         pint \
         graphviz \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         coveralls \
         codecov
     - sudo make -j NOCUDA=1

Dockerfile.cpu

@@ -13,6 +13,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Update ctypesgen
+RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py
+RUN pip --no-cache-dir install \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f
+
 # Build the library
 WORKDIR /bifrost
 COPY . .

Dockerfile.gpu

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:8.0
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 MAINTAINER Ben Barsdell <benbarsdell@gmail.com>
 
@@ -32,7 +32,7 @@ RUN pip --no-cache-dir install \
         contextlib2 \
         simplejson \
         pint \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         graphviz
 
 ENV TERM xterm

Dockerfile_prereq.gpu

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:8.0
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 MAINTAINER Ben Barsdell <benbarsdell@gmail.com>
 
@@ -31,7 +31,7 @@ RUN pip --no-cache-dir install \
         contextlib2 \
         simplejson \
         pint \
-        git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32 \
+        git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f \
         graphviz
 
 ENV TERM xterm

README.md

@@ -99,7 +99,7 @@ print "All done"
  * ctypesgen
 
 ```
-$ sudo pip install numpy contextlib2 pint git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32
+$ sudo pip install numpy contextlib2 pint git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f
 ```
 
 ### Bifrost installation

config.mk

@@ -28,7 +28,7 @@ endif
 BIFROST_NAME          = bifrost
 LIBBIFROST_NAME       = lib$(BIFROST_NAME)
 LIBBIFROST_MAJOR      = 0
-LIBBIFROST_MINOR      = 8
+LIBBIFROST_MINOR      = 9
 LIBBIFROST_PATCH      = 0
 LIBBIFROST_SO         = $(LIBBIFROST_NAME)$(SO_EXT)
 LIBBIFROST_SO_MAJ     = $(LIBBIFROST_SO).$(LIBBIFROST_MAJOR)

docs/source/Getting-started-guide.rst

@@ -44,7 +44,7 @@ numpy, matplotlib, contextlib2, simplejson, pint, graphviz, ctypesgen
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you have already installed pip, this step should be as simple as
-``pip install --user numpy matplotlib contextlib2 simplejson pint graphviz git+https://github.com/davidjamesca/ctypesgen.git@3d2d9803339503d2988382aa861b47a6a4872c32``.
+``pip install --user numpy matplotlib contextlib2 simplejson pint graphviz git+https://github.com/olsonse/ctypesgen.git@9bd2d249aa4011c6383a10890ec6f203d7b7990f``.
 
 C++ dependencies
 ~~~~~~~~~~~~~~~~

python/Makefile

@@ -21,7 +21,7 @@ $(BIFROST_PYTHON_VERSION_FILE): ../config.mk
 	@echo "__version__ = \"$(LIBBIFROST_MAJOR).$(LIBBIFROST_MINOR).$(LIBBIFROST_PATCH)\"" > $@
 
 define run_ctypesgen
-	ctypesgen.py -l$1 -I$2 $^ -o $@
+	python -c 'from ctypesgen import main as ctypeswrap; ctypeswrap.main()' -l$1 -I$2 $^ -o $@
 	# WAR for 'const char**' being generated as POINTER(POINTER(c_char)) instead of POINTER(c_char_p)
 	sed -i 's/POINTER(c_char)/c_char_p/g' $@
 	# WAR for a buggy WAR in ctypesgen that breaks type checking and auto-byref functionality

python/bifrost/DataType.py

@@ -1,5 +1,5 @@
 
-# Copyright (c) 2016, The Bifrost Authors. All rights reserved.
+# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -38,7 +38,15 @@
 cf32: 32+32-bit complex floating point
 """
 
-from libbifrost import _bf
+# Python2 compatibility
+from __future__ import division, absolute_import
+import sys
+string_types = (str,)
+if sys.version_info < (3,):
+    range = xrange
+    string_types = (basestring,)
+
+from bifrost.libbifrost import _bf
 import numpy as np
 
 # Custom dtypes to represent additional complex types
@@ -100,15 +108,15 @@ def is_vector_structure(dtype):
     if dtype.names is None:
         return False
     ndim = len(dtype.names)
-    vector_field_names = tuple('f%i' % i for i in xrange(ndim))
+    vector_field_names = tuple('f%i' % i for i in range(ndim))
     return (dtype.kind == 'V' and
             dtype.names == vector_field_names and
-            all([dtype[i] == dtype[0] for i in xrange(1, ndim)]))
+            all([dtype[i] == dtype[0] for i in range(1, ndim)]))
 
 class DataType(object):
     # Note: Default of None results in default Numpy type (np.float)
     def __init__(self, t=None):
-        if isinstance(t, basestring):
+        if isinstance(t, string_types):
             for i, char in enumerate(t):
                 if char.isdigit():
                     break
@@ -148,10 +156,10 @@ def __init__(self, t=None):
                 t = t[0]
             self._kind = t.kind
             if t.kind == 'c':
-                self._nbit /= 2   # Bifrost convention is nbit per real component
+                self._nbit //= 2   # Bifrost convention is nbit per real component
                 self._kind = 'cf' # Numpy only supports floating-point complex types
             elif t.kind == 'V': # WAR to support custom integer complex types
-                self._nbit /= 2
+                self._nbit //= 2
                 if t in [ci4, ci8, ci16, ci32, ci64]:
                     self._kind = 'ci'
                 elif t in [cf16]:

python/bifrost/GPUArray.py

@@ -28,10 +28,9 @@
 
 import ctypes
 import numpy as np
-from memory import raw_malloc, raw_free, memset, memcpy, memcpy2D
-from libbifrost import _check, _string2space
-from bifrost.libbifrost import _bf
-from array import _array2bifrost
+from bifrost.memory import raw_malloc, raw_free, memset, memcpy, memcpy2D
+from bifrost.libbifrost import _bf, _check, _string2space
+from bifrost.array import _array2bifrost # This doesn't exist!
 
 class GPUArray(object):
     def __init__(self, shape, dtype, buffer=None, offset=0, strides=None):

python/bifrost/Space.py

@@ -1,5 +1,5 @@
 
-# Copyright (c) 2016, The Bifrost Authors. All rights reserved.
+# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -41,7 +41,7 @@
 
 class Space(object):
     def __init__(self, s):
-        if isinstance(s, basestring):
+        if isinstance(s, str):
             if s not in set(['auto', 'system',
                              'cuda', 'cuda_host', 'cuda_managed']):
                 raise ValueError('Invalid space: %s' % s)

python/bifrost/__init__.py

@@ -31,30 +31,33 @@
 """
 
 # TODO: Decide how to organise the namespace
-import core, memory, affinity, ring, block, address, udp_socket
-import pipeline
-import device
-from ndarray import ndarray, asarray, empty_like, empty, zeros_like, zeros
-import views
-from map import map
-from pipeline import Pipeline, get_default_pipeline, block_scope
-import blocks
-from block_chainer import BlockChainer
-from reduce import reduce
+
+from __future__ import print_function, absolute_import
+
+from bifrost import core, memory, affinity, ring, block, address, udp_socket
+from bifrost import pipeline
+from bifrost import device
+from bifrost.ndarray import ndarray, asarray, empty_like, empty, zeros_like, zeros
+from bifrost import views
+from bifrost.map import map
+from bifrost.pipeline import Pipeline, get_default_pipeline, block_scope
+from bifrost import blocks
+from bifrost.block_chainer import BlockChainer
+from bifrost.reduce import reduce
 # import copy_block, transpose_block, scrunch_block, sigproc_block, fdmt_block
-# from transpose import transpose
-# from unpack import unpack
-# from quantize import quantize
+# from bifrost.transpose import transpose
+# from bifrost.unpack import unpack
+# from bifrost.quantize import quantize
 
 try:
-    from .version import __version__
+    from bifrost.version import __version__
 except ImportError:
-    print "*************************************************************************"
-    print "Please run `make` from the root of the source tree to generate version.py"
-    print "*************************************************************************"
+    print("*************************************************************************")
+    print("Please run `make` from the root of the source tree to generate version.py")
+    print("*************************************************************************")
     raise
 __author__     = "The Bifrost Authors"
-__copyright__  = "Copyright (c) 2016, The Bifrost Authors. All rights reserved.\nCopyright (c) 2016, NVIDIA CORPORATION. All rights reserved."
+__copyright__  = "Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved.\nCopyright (c) 2016, NVIDIA CORPORATION. All rights reserved."
 __credits__    = ["Ben Barsdell"]
 __license__    = "BSD 3-Clause"
 __maintainer__ = "Ben Barsdell"

python/bifrost/addon/leda/bandfiles.py

@@ -24,6 +24,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import os, sys
 
 sys.path.append('..')
@@ -83,12 +85,12 @@ def __init__(self, fname):
     obs1 = extract_obs_offset_from_name(fname)
     obs2 = extract_obs_offset_in_file(fname)
     if obs1 != obs2 and obs1 != "UNKNOWN" and obs2 !="UNKNOWN":
-      print "Consistency Error", fname, ": OBS_OFFSET in file doesn't match the offset in the name"
+      print("Consistency Error", fname, ": OBS_OFFSET in file doesn't match the offset in the name")
     """
 
     if hedr["SOURCE"] == "LEDA_TEST":
-      if not is_integer(n_scans): print "CONSISTENCY ERROR, ", fname, "scan:",n_scans, "is not integer"
-      if not is_integer((self.end_time-self.start_time)/9.0): print "CONSISTENCY ERROR", fname, ": not 9 sec dump in file"
+      if not is_integer(n_scans): print("CONSISTENCY ERROR, ", fname, "scan:",n_scans, "is not integer")
+      if not is_integer((self.end_time-self.start_time)/9.0): print("CONSISTENCY ERROR", fname, ": not 9 sec dump in file")
 
 # Gather the file info for all files that have different frequency but the same observation time.
 class BandFiles(object):
@@ -106,7 +108,7 @@ def __init__(self, basename):
     if basename[-5:] == ".dada":                # Just a single file
       if os.access(basename,os.R_OK): self.files.append(FileInfo(basename))
       else: "Error:", basename, "does not exist or is not readable"
-      print basename, FileInfo(basename), self.files
+      print(basename, FileInfo(basename), self.files)
     else:
 
       # Look for files in all the standard locations
@@ -138,7 +140,7 @@ def __init__(self, basename):
       if f.start_time not in self.start_time_present: self.start_time_present.append(f.start_time)
 
     if len(self.start_time_present) > 1:
-      print "Error: Files with same timestamp in their name have different internal time. Basename:",basename
+      print("Error: Files with same timestamp in their name have different internal time. Basename:",basename)
       #sys.exit(1)
 
     self.start_time = self.start_time_present[0]

python/bifrost/addon/leda/blocks.py

@@ -1,4 +1,5 @@
-# Copyright (c) 2016, The Bifrost Authors. All rights reserved.
+
+# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -28,6 +29,12 @@
 This file contains blocks specific to LEDA-OVRO.
 """
 
+# Python2 compatibility
+from __future__ import print_function
+import sys
+if sys.version_info < (3,):
+    range = xrange
+
 import os
 import bandfiles
 import bifrost
@@ -75,10 +82,10 @@ def __init__(self, time_stamp, core=-1, gulp_nframe=4096):
         i += 1
 
     # Report what we've got
-    print "Num files in time:",  len(beamformer_scans)
-    print "File and number:"
+    print("Num files in time:",  len(beamformer_scans))
+    print("File and number:")
     for scan in beamformer_scans:
-      print os.path.basename(scan.files[0].name)+":", len(scan.files)
+      print(os.path.basename(scan.files[0].name)+":", len(scan.files))
 
     self.beamformer_scans = beamformer_scans     # List of full-band time steps
 
@@ -100,7 +107,7 @@ def main(self, input_rings, output_rings):
     ohdr["tsamp"] = self.SAMPLING_RATE
     ohdr['foff'] = self.CHANNEL_WIDTH
 
-    #print length_one_second, ring_span_size, file_chunk_size, number_of_chunks
+    #print(length_one_second, ring_span_size, file_chunk_size, number_of_chunks)
 
     with self.oring.begin_writing() as oring:
 
@@ -109,7 +116,7 @@ def main(self, input_rings, output_rings):
         # Go through the frequencies
         for f in scan.files:
 
-          print "Opening", f.name
+          print("Opening", f.name)
 
           with open(f.name,'rb') as ifile:
             ifile.read(self.HEADER_SIZE)
@@ -126,10 +133,10 @@ def main(self, input_rings, output_rings):
                 try:
                     data = np.fromfile(ifile, count=file_chunk_size, dtype=np.int8).astype(np.float32)
                 except:
-                    print "Bad read. Stopping read."
+                    print("Bad read. Stopping read.")
                     return
                 if data.size != length_one_second*self.N_BEAM*self.N_CHAN*2:
-                    print "Bad data shape. Stopping read."
+                    print("Bad data shape. Stopping read.")
                     return
                 data = data.reshape(length_one_second, self.N_BEAM, self.N_CHAN, 2)
                 power = (data[...,0]**2 + data[...,1]**2).mean(axis=1)  # Now have time by frequency.