Move ffi and mpi code to deprecated

ffiwrappers/src/arlwrap.py → deprecated_code/ffiwrappers/src/arlwrap.py

@@ -3,7 +3,6 @@
 # In this file we declare the functions that need to be exposed to C (wrapped) --with the callback modifier
 # and the needed structs -- with cdef
 
-import numpy
 import collections
 import sys
 
@@ -27,7 +26,7 @@
 from workflows.serial.pipelines.pipeline_serial import ical_list_serial_workflow
 from data_models.data_model_helpers import export_image_to_hdf5
 
-from ffiwrappers.src.arlwrap_support import *
+from src.arlwrap_support import *
 
 import logging
 import os

deprecated_code/setup.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+#from setuptools import setup
+from setuptools import setup, Extension
+from distutils.sysconfig import get_config_var, get_config_vars
+from setuptools.command.build_ext import build_ext
+from subprocess import call
+import os
+import sys
+import glob
+
+# Bail on Python < 3
+assert sys.version_info[0] >= 3
+
+
+# MF. This is a workaround to be able to build the library with MacOS
+if sys.platform == 'darwin':
+	vars = get_config_vars()
+	vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle','-dynamiclib')
+	os.environ["CC"] = "clang"
+
+
+
+
+# NB. These are not really Python extensions (i.e., they do not
+# Py_Initialize() and they do define main() ), we are just cheating to
+# re-use the setuptools build support.
+
+libarlffi = Extension('libarlffi',
+                   sources = ['ffiwrappers/src/arlwrap.c', 'ffiwrappers/src/wrap_support.c', 'ffiwrappers/src/wrappingcore.c'],
+                   undef_macros = ['NDEBUG'],
+                   extra_compile_args = ['-Wno-strict-prototypes'],
+                   libraries= ['cfitsio'],
+		   optional=True,
+)
+
+packages = ['data_models', 'processing_library', 'processing_components', 'workflows', 'wrappers']
+package_data = [i for p in packages for i in glob.glob(p+'/*/') + glob.glob(p+'/*/*/')]
+setup(name='algorithm-reference-library',
+      version='0.9',
+      python_requires='>=3',
+      description='Algorithm Reference Library for Radio Interferometry',
+      long_description=open('README.md').read(),
+      author='Tim Cornwell, Peter Wortmann, Bojan Nikolic, Montse Farreras',
+      author_email='realtimcornwell@gmail.com',
+      url='https://github.com/SKA-ScienceDataProcessor/algorithm-reference-library',
+      license='Apache License Version 2.0',
+      packages=(packages + package_data),
+      test_suite="tests",
+      tests_require=['pytest'],
+      ext_modules = [libarlffi]
+      )
+
+
+# noinspection PyCompatibility
+class CFFIBuild(build_ext):
+        def run(self):
+            super().run()
+
+            cmd = [
+                    "make",
+                    "BUILD_LIB={}".format(self.build_lib)
+                ]
+
+            call(cmd)
+
+#setup(name='FFI_Demo',
+#    version='0.1',
+#    python_requires='>=3',
+#    description = "Demo for compilation of FFI-wrapped Python callable from C",
+#    ext_modules = [libarlffi])
+#,
+#    cmdclass = {'build_ext': CFFIBuild})

workflows/mpi/calibration/calibration_mpi.py → deprecated_code/workflows/mpi/calibration/calibration_mpi.py

@@ -4,12 +4,11 @@
 
 import numpy
 
-from data_models.data_model_helpers import GainTable
-from wrappers.mpi.calibration.calibration_control import calibrate_function, apply_calibration_function, \
+from wrappers.mpi.calibration.calibration_control import apply_calibration_function, \
     solve_calibrate_function
-from wrappers.mpi.visibility.coalesce import convert_visibility_to_blockvisibility
-from wrappers.mpi.visibility.gather_scatter import visibility_gather_channel
-from wrappers.mpi.visibility.operations import integrate_visibility_by_channel, \
+from visibility import convert_visibility_to_blockvisibility
+from visibility import visibility_gather_channel
+from visibility.operations import integrate_visibility_by_channel, \
     divide_visibility
 
 from mpi4py import MPI

workflows/mpi/imaging/imaging_mpi.py → deprecated_code/workflows/mpi/imaging/imaging_mpi.py

@@ -13,23 +13,19 @@
 
 from data_models.memory_data_models import Image, Visibility
 from data_models.parameters import get_parameter
-from processing_library.image.operations import copy_image, create_empty_image_like
-from workflows.shared.imaging.imaging_shared import imaging_context
-from workflows.shared.imaging.imaging_shared import sum_invert_results, remove_sumwt, sum_predict_results, \
-    threshold_list
+from processing_library.image.operations import create_empty_image_like
+from workflows.shared.imaging.imaging_shared import sum_invert_results, remove_sumwt, sum_predict_results
 from wrappers.mpi.griddata.gridding import grid_weight_to_griddata, griddata_reweight, griddata_merge_weights
 from wrappers.mpi.griddata.kernels import create_pswf_convolutionfunction
 from wrappers.mpi.griddata.operations import create_griddata_from_image
 from wrappers.mpi.image.deconvolution import deconvolve_cube, restore_cube
-from wrappers.mpi.image.gather_scatter import image_scatter_facets, image_gather_facets, \
+from wrappers.mpi.image.gather_scatter import image_gather_facets, \
     image_scatter_channels, image_gather_channels
 from wrappers.mpi.image.operations import calculate_image_frequency_moments
-from wrappers.mpi.visibility.base import copy_visibility
-from wrappers.mpi.visibility.gather_scatter import visibility_scatter, visibility_gather
+from visibility import copy_visibility
 from wrappers.mpi.imaging.weighting import taper_visibility_gaussian, taper_visibility_tukey
 
 from mpi4py import MPI
-import sys
 
 log = logging.getLogger(__name__)
 

setup.py

@@ -1,73 +1,38 @@
 #!/usr/bin/env python
 
-#from setuptools import setup
-from setuptools import setup, Extension
-from distutils.sysconfig import get_config_var, get_config_vars
-from setuptools.command.build_ext import build_ext
-from subprocess import call
+import glob
 import os
 import sys
-import glob
+from distutils.sysconfig import get_config_vars
+
+# from setuptools import setup
+from setuptools import setup
 
 # Bail on Python < 3
 assert sys.version_info[0] >= 3
 
-
 # MF. This is a workaround to be able to build the library with MacOS
 if sys.platform == 'darwin':
-	vars = get_config_vars()
-	vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle','-dynamiclib')
-	os.environ["CC"] = "clang"
-
-
-
+    vars = get_config_vars()
+    vars['LDSHARED'] = vars['LDSHARED'].replace('-bundle', '-dynamiclib')
+    os.environ["CC"] = "clang"
 
 # NB. These are not really Python extensions (i.e., they do not
 # Py_Initialize() and they do define main() ), we are just cheating to
 # re-use the setuptools build support.
 
-libarlffi = Extension('libarlffi',
-                   sources = ['ffiwrappers/src/arlwrap.c', 'ffiwrappers/src/wrap_support.c', 'ffiwrappers/src/wrappingcore.c'],
-                   undef_macros = ['NDEBUG'],
-                   extra_compile_args = ['-Wno-strict-prototypes'],
-                   libraries= ['cfitsio'],
-		   optional=True,
-)
-
 packages = ['data_models', 'processing_library', 'processing_components', 'workflows', 'wrappers']
-package_data = [i for p in packages for i in glob.glob(p+'/*/') + glob.glob(p+'/*/*/')]
+package_data = [i for p in packages for i in glob.glob(p + '/*/') + glob.glob(p + '/*/*/')]
 setup(name='algorithm-reference-library',
       version='0.9',
       python_requires='>=3',
       description='Algorithm Reference Library for Radio Interferometry',
       long_description=open('README.md').read(),
-      author='Tim Cornwell, Peter Wortmann, Bojan Nikolic, Montse Farreras',
+      author='Tim Cornwell, Peter Wortmann, Bojan Nikolic, Montse Farreras, Feng Wang',
       author_email='realtimcornwell@gmail.com',
       url='https://github.com/SKA-ScienceDataProcessor/algorithm-reference-library',
       license='Apache License Version 2.0',
       packages=(packages + package_data),
       test_suite="tests",
-      tests_require=['pytest'],
-      ext_modules = [libarlffi]
+      tests_require=['pytest']
       )
-
-
-# noinspection PyCompatibility
-class CFFIBuild(build_ext):
-        def run(self):
-            super().run()
-
-            cmd = [
-                    "make",
-                    "BUILD_LIB={}".format(self.build_lib)
-                ]
-
-            call(cmd)
-
-#setup(name='FFI_Demo',
-#    version='0.1',
-#    python_requires='>=3',
-#    description = "Demo for compilation of FFI-wrapped Python callable from C",
-#    ext_modules = [libarlffi])
-#,
-#    cmdclass = {'build_ext': CFFIBuild})

workflows/__init__.py

@@ -1,4 +1,4 @@
 """ Algorithm Reference Library workflows
 
 """
-__all__ = ['arlexecute', 'serial', 'ffiwrapped', 'mpi', 'scripts', 'shared']
+__all__ = ['arlexecute', 'serial', 'scripts', 'shared']

workflows/scripts/pipelines/performance/pipelines-timings/pipelines_mpi_timings.py

@@ -16,11 +16,9 @@
 from processing_library.image.operations import create_image
 from processing_library.util.sizeof import get_size
 from workflows.mpi.imaging.imaging_mpi import invert_list_mpi_workflow, \
-    weight_list_mpi_workflow, predict_list_mpi_workflow, \
-    taper_list_mpi_workflow, remove_sumwt
-from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow, \
-    weight_list_serial_workflow, predict_list_serial_workflow, \
-    taper_list_serial_workflow
+    predict_list_mpi_workflow, \
+    remove_sumwt
+from workflows.serial.imaging.imaging_serial import weight_list_serial_workflow, taper_list_serial_workflow
 from workflows.mpi.pipelines.pipeline_mpi import ical_list_mpi_workflow
 #from workflows.mpi.simulation.simulation_mpi import simulate_list_mpi_workflow, \
 #    corrupt_list_mpi_workflow
@@ -33,8 +31,8 @@
 from wrappers.mpi.image.gather_scatter import image_gather_channels
 from wrappers.mpi.image.operations import export_image_to_fits, qa_image
 from wrappers.mpi.imaging.base import advise_wide_field
-from wrappers.mpi.simulation.testing_support import create_low_test_skymodel_from_gleam
-from wrappers.mpi.visibility.coalesce import convert_blockvisibility_to_visibility
+from simulation import create_low_test_skymodel_from_gleam
+from visibility import convert_blockvisibility_to_visibility
 
 pp = pprint.PrettyPrinter()