My attempts at fixing conda builds. Now the file name for the C exten…

…sion contains the SOABI name (with the cpython version). The install name for the library is updated to contain the SOABI name

common.mk

@@ -27,6 +27,8 @@ PYTHON_CONFIG_EXE:=
 ## Set OpenMP for both theory and mocks
 OPT += -DUSE_OMP
 
+CONDA_BUILD ?=0
+
 ### You should NOT edit below this line
 DISTNAME:=Corrfunc
 MAJOR:=2
@@ -379,15 +381,27 @@ ifeq ($(DO_CHECKS), 1)
       export PYTHON_LIBDIR := $(shell $(PYTHON_CONFIG_EXE) --prefix)/lib
       export PYTHON_LIBS   := $(shell $(PYTHON_CONFIG_EXE) --libs)
       export PYTHON_LINK   := -L$(PYTHON_LIBDIR) $(PYTHON_LIBS) -Xlinker -rpath -Xlinker $(PYTHON_LIBDIR)
+      SOABI := $(shell $(PYTHON) -c "from __future__ import print_function; import sysconfig; print(sysconfig.get_config_var('SOABI'))")
+      ifndef SOABI
+        PYTHON_SOABI = 
+      else
+        PYTHON_SOABI = .$(SOABI)
+      endif
+      export PYTHON_SOABI
       export PYTHON_LIB_BASE := $(strip $(subst -l,lib, $(filter -lpython%,$(PYTHON_LIBS))))
 
       ### Check if conda is being used on OSX - then we need to fix python link libraries
       export FIX_PYTHON_LINK := 0
-      ifeq ($(UNAME), Darwin)
-        PATH_TO_PYTHON := $(shell which python)
-        ifeq (conda, $(findstring conda, $(PATH_TO_PYTHON)))
-	  FIX_PYTHON_LINK := 1
+      ifeq ($(CONDA_BUILD), 0)
+        ## Check if conda build is under progress -> do nothing in that case. Let conda handle it
+        ifeq ($(UNAME), Darwin)
+          PATH_TO_PYTHON := $(shell which python)
+          ifeq (conda, $(findstring conda, $(PATH_TO_PYTHON)))
+	    FIX_PYTHON_LINK := 1
+          endif
         endif
+      endif
+      ifeq ($(UNAME), Darwin)
         PYTHON_LINK := $(filter-out -framework, $(PYTHON_LINK))
         PYTHON_LINK := $(filter-out -ldl, $(PYTHON_LINK))
         PYTHON_LINK := $(filter-out CoreFoundation, $(PYTHON_LINK))

meta.yaml

@@ -1,31 +1,35 @@
 package:
   name: corrfunc
-  version: "0.2.1"
+  version: "2.0.0"
 
 source:
   path: dist
-  fn: Corrfunc-0.2.1.tar.gz
-  md5: 54e2b777517f074847f11223344a1577
-
-  # fn: Corrfunc-0.2.1.tar.gz
-  # url: https://github.com/manodeep/Corrfunc/archive/0.2.1.tar.gz
-  # md5: 2bd15dac1706cb59b5ddb528cd5c7db2
+  fn: Corrfunc-2.0.0.tar.gz
+  md5: 59ad43b044225ce8c87e60f41dda6873
 
+
+  # git_url: https://github.com/manodeep/Corrfunc.git
+  # git_rev: master
+
 requirements:
   build:
-    - python
-    - numpy
-    - setuptools 
+    - python >=2.6
+    - numpy >=1.7
+    - setuptools
+    - future
+    - gsl
 
   run:
-    - python
-    - numpy
+    - python >=2.6
+    - future
+    - numpy >=1.7
+    - gsl
 
 build:
   detect_binary_files_with_prefix: True
-  script: make install && python setup.py install 
+  script: python setup.py install --single-version-externally-managed --record record.txt
   binary_relocation: True
-  skip: True #[win]
+  skip: True # [win]
 
 test:
   files:

mocks/python_bindings/Makefile

@@ -3,6 +3,9 @@ UTILS_DIR := $(ROOT_DIR)/utils
 IO_DIR := $(ROOT_DIR)/io
 LIB_DIR := $(ROOT_DIR)/lib
 DATA_DIR := ../tests/data
+INSTALL_HEADERS_DIR := $(ROOT_DIR)/include
+INSTALL_LIB_DIR := $(ROOT_DIR)/lib
+INSTALL_BIN_DIR := $(ROOT_DIR)/bin
 
 MOCKS_DIR := $(ROOT_DIR)/mocks
 DDrppi_mocks_DIR := $(MOCKS_DIR)/DDrppi
@@ -16,7 +19,7 @@ VPF_mocks_LIB := countspheres_mocks
 include $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk
 
 PROJECT := _countpairs_mocks
-PYTHON_LIBRARY := $(PROJECT).so.$(MAJOR).$(MINOR).$(PATCHLEVEL)
+PYTHON_EXTN := $(PROJECT).so.$(MAJOR).$(MINOR).$(PATCHLEVEL)
 SOURCES := $(PROJECT).c
 OBJECTS := $(SOURCES:.c=.o)
 C_LIBRARIES := $(DDrppi_mocks_DIR)/lib$(DDrppi_mocks_LIB).a $(WTHETA_DIR)/lib$(WTHETA_LIB).a $(VPF_mocks_DIR)/lib$(VPF_mocks_LIB).a
@@ -31,31 +34,24 @@ LIB_INCLUDE:=-I$(DDrppi_mocks_DIR) -I$(WTHETA_DIR) -I$(VPF_mocks_DIR)
 
 
 ifeq ($(UNAME), Darwin)
-  LINKER_FLAGS := -install_name "$(PYTHON_LIBRARY)"
+  LINKER_FLAGS := -install_name "$(PYTHON_EXTN)"
 else
   LINKER_FLAGS := -Wl,-soname=$(PROJECT).$(MAJOR)
 endif
 
 all: sharedlib $(LIBRARY) $(SOURCES) $(C_LIBRARIES) $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk Makefile 
 
-$(PROJECT).so: $(PYTHON_LIBRARY)
+$(PROJECT).so: $(PYTHON_EXTN)
 	$(RM) $(PROJECT).so
-	ln -s $(PYTHON_LIBRARY) $(PROJECT).so
+	ln -s $(PYTHON_EXTN) $(PROJECT).so
 
 $(PROJECT).o: $(PROJECT).c $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk Makefile $(C_LIBRARIES) $(INCL)
 	$(CC) $(CFLAGS) $(LIB_INCLUDE) $(PYTHON_CFLAGS)  $(INCLUDE) $< -c -o $@
 
 tests: sharedlib $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk Makefile $(C_LIBRARIES) $(OBJECTS) | $(ROOT_DIR)/lib
 	python call_correlation_functions_mocks.py
 
-$(LIB_DIR)/$(PYTHON_LIBRARY): $(PYTHON_LIBRARY) | $(LIB_DIR)
-	cp -p $(PYTHON_LIBRARY) $(LIB_DIR)/
-
-$(LIB_DIR)/$(PROJECT).so:$(LIB_DIR)/$(PYTHON_LIBRARY)
-	$(RM) $(LIB_DIR)/$(PROJECT).so
-	cd $(LIB_DIR) && ln -s $(PYTHON_LIBRARY) $(PROJECT).so
-
-$(PYTHON_LIBRARY): $(OBJECTS) $(C_LIBRARIES) $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk Makefile 
+$(PYTHON_EXTN): $(OBJECTS) $(C_LIBRARIES) $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk Makefile 
 	$(CC) $(OBJECTS) $(C_LIBRARIES) $(LINKER_FLAGS) $(CLINK) $(GSL_LINK) $(PYTHON_LINK) -shared -o $@
 
 $(DDrppi_mocks_DIR)/lib$(DDrppi_mocks_LIB).a: $(DDrppi_mocks_DIR)/*.c $(DDrppi_mocks_DIR)/*.c.src $(DDrppi_mocks_DIR)/*.h.src $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk
@@ -67,27 +63,34 @@ $(WTHETA_DIR)/lib$(WTHETA_LIB).a: $(WTHETA_DIR)/*.c $(WTHETA_DIR)/*.c.src $(WTHE
 $(VPF_mocks_DIR)/lib$(VPF_mocks_LIB).a: $(VPF_mocks_DIR)/*.c $(VPF_mocks_DIR)/*.c.src $(VPF_mocks_DIR)/*.h.src $(ROOT_DIR)/mocks.options $(ROOT_DIR)/common.mk
 	$(MAKE) -C $(VPF_mocks_DIR) libs
 
-sharedlib: $(PROJECT).so | $(LIB_DIR)
+sharedlib: $(PROJECT).so
 ifeq ($(FIX_PYTHON_LINK), 1)
 	@{ \
-		CURRENT_PYTHON_LIB=`otool -L $(PYTHON_LIBRARY) | grep -i python | cut -d " " -f1 | xargs` ; \
-                PYTHON_LIB_NAME=$(PYTHON_LIB_BASE).dylib ; \
+		CURRENT_PYTHON_LIB=`otool -L $(PYTHON_EXTN) | grep -i python | cut -d " " -f1 | grep -v $(PROJECT) | xargs` ; \
+		PYTHON_LIB_NAME=$(PYTHON_LIB_BASE).dylib ; \
 		LINK_PYTHON_LIB=$(PYTHON_LIBDIR)/$$PYTHON_LIB_NAME ;\
 		if [[ "$$CURRENT_PYTHON_LIB" != "$$LINK_PYTHON_LIB" ]] ; then \
-		install_name_tool -change $$CURRENT_PYTHON_LIB $$LINK_PYTHON_LIB $(PYTHON_LIBRARY); \
+		install_name_tool -change $$CURRENT_PYTHON_LIB $$LINK_PYTHON_LIB $(PYTHON_EXTN); \
 		fi ;\
 	}
 endif
 
-install: sharedlib $(LIB_DIR)/$(PYTHON_LIBRARY) 
+install: sharedlib $(LIB_DIR)/$(PYTHON_EXTN) 
 
 .PHONY: sharedlib
 
+$(INSTALL_LIB_DIR)/$(PYTHON_EXTN): $(PYTHON_EXTN) | $(INSTALL_LIB_DIR)
+	cp -p $(PYTHON_EXTN) $(INSTALL_LIB_DIR)/
+
+$(INSTALL_LIB_DIR)/$(PROJECT).so:$(INSTALL_LIB_DIR)/$(PYTHON_EXTN)
+	$(RM) $(INSTALL_LIB_DIR)/$(PROJECT).so
+	cd $(INSTALL_LIB_DIR) && ln -s $(PYTHON_EXTN) $(PROJECT).so
+
+
 distclean: clean | $(LIB_DIR)
-	cd $(LIB_DIR) && $(RM) $(PYTHON_LIBRARY)
-	cd $(LIB_DIR) && $(RM) $(PROJECT).so
+	cd $(INSTALL_LIB_DIR) && $(RM) $(PYTHON_EXTN) $(PROJECT).so
 
 clean:
-	$(RM) $(PYTHON_LIBRARY) $(OBJECTS) $(PROJECT).so
+	$(RM) $(PYTHON_EXTN) $(OBJECTS) $(PROJECT).so
 
 

mocks/wtheta/countpairs_theta_mocks_impl.c.src

@@ -143,9 +143,6 @@ countpairs_theta_mocks_func_ptr_DOUBLE countpairs_theta_mocks_driver_DOUBLE(cons
               __FUNCTION__, function_dispatch, num_functions);
       return NULL;
     }
-    if(function_dispatch == fallback_offset) {
-        fprintf(stderr,"RETURNING FALLBACK FUNCTION\n");
-    }
     function = allfunctions[function_dispatch];
     old_isa = options->instruction_set;
 

rules.mk

@@ -75,12 +75,15 @@ $(INSTALL_LIB_DIR) $(INSTALL_BIN_DIR) $(INSTALL_HEADERS_DIR):
 
 ifdef PROJECT
   ifeq ($(UNAME), Darwin)
-    PYTHON_LINK += -Xlinker -install_name -Xlinker $(PROJECT)
+    PYTHON_LINK += -Xlinker -install_name -Xlinker $(PROJECT)$(PYTHON_SOABI)
   else
-    PYTHON_LINK += -Xlinker -soname -Xlinker $(PROJECT).so.$(MAJOR)
+    PYTHON_LINK += -Xlinker -soname -Xlinker $(PROJECT)$(PYTHON_SOABI).so.$(MAJOR)
   endif
+
 endif
 
+
+
 .PHONY: clean clena celan install lib tests distclean all realclean libs
 
 celna: clean

setup.py

@@ -458,6 +458,13 @@ def setup_packages():
     # Some command options require headers/libs to be generated
     # so that the following dirs_patters supplies them.
     if install_required():
+        from distutils.sysconfig import get_config_var
+        if get_config_var('SHLIB_EXT') != '".so"' and version_info[0] == 2:
+            msg = "The extensions all get the `.so` automatically. "\
+                  "However, python expects the extension to be `{0}`"\
+                  .format(get_config_var('SHLIB_EXT'))
+            raise ValueError(msg)
+
         # global variable compiler is set if passed in
         # command-line
         extra_string = ''
@@ -466,6 +473,15 @@ def setup_packages():
 
         command = "make libs {0}".format(extra_string)
         run_command(command)
+
+    else:
+        # not installing. Check if creating source distribution
+        # in that case run distclean to delete auto-generated C
+        # files
+        if 'sdist' in sys.argv:
+            command = "make distclean"
+            run_command(command)
+
 
     # find all the data-files required.
     # Now the lib + associated header files have been generated
@@ -509,34 +525,37 @@ def setup_packages():
                    'Programming Language :: Python :: 3.4',
                    'Programming Language :: Python :: 3.5']
     metadata = dict(
-        name=projectname,
-        version=version,
-        author='Manodeep Sinha',
-        author_email='manodeep@gmail.com',
-        maintainer='Manodeep Sinha',
-        maintainer_email='manodeep@gmail.com',
-        url=base_url,
-        download_url='{0}/archive/{1}-{2}.tar.gz'.format(
-            base_url, projectname, version),
-        description='Blazing fast correlation functions on the CPU',
-        long_description=long_description,
-        classifiers=classifiers,
-        license='MIT',
-        # Solaris might work, Windows will almost certainly not work
-        platforms=["Linux", "Mac OSX", "Unix"],
-        keywords=['correlation functions', 'simulations',
-                  'surveys', 'galaxies'],
-        provides=[projectname],
-        packages=find_packages(),
-        ext_package=projectname,
-        ext_modules=extensions,
-        package_data={'': data_files},
-        include_package_data=True,
-        install_requires=['setuptools',
-                          'numpy>={0}.{1}'.format(min_np_major, min_np_minor),
-                          'future'],
-        zip_safe=False,
-        cmdclass={'build_ext': BuildExtSubclass})
+            name=projectname,
+            version=version,
+            author='Manodeep Sinha',
+            author_email='manodeep@gmail.com',
+            maintainer='Manodeep Sinha',
+            maintainer_email='manodeep@gmail.com',
+            url=base_url,
+            download_url='{0}/archive/{1}-{2}.tar.gz'.format(
+                    base_url, projectname, version),
+            description='Blazing fast correlation functions on the CPU',
+            long_description=long_description,
+            classifiers=classifiers,
+            license='MIT',
+            # Solaris might work, Windows will almost certainly not work
+            platforms=["Linux", "Mac OSX", "Unix"],
+            keywords=['correlation functions', 'simulations',
+                      'surveys', 'galaxies'],
+            provides=[projectname],
+            packages=find_packages(),
+            ext_package=projectname,
+            ext_modules=extensions,
+            package_data={'': data_files},
+            include_package_data=True,
+            setup_requires=['setuptools',
+                            'numpy>={0}.{1}'.format(min_np_major,
+                                                    min_np_minor)],
+            install_requires=['numpy>={0}.{1}'.format(min_np_major,
+                                                      min_np_minor),
+                              'future'],
+            zip_safe=False,
+            cmdclass={'build_ext': BuildExtSubclass})
 
     # Now the actual setup
     try:

theory/README.md

@@ -1,5 +1,4 @@
 /* Author: Manodeep Sinha <manodeep@gmail.com>
-	 Date: At some point in early 2015. 
 	 LICENSE: MIT 
 */
 
@@ -8,48 +7,16 @@ three correlation measures in a cosmological box:
 
 1. Measures auto/cross-correlations between two boxes. 
 Can be used to make full 3-d correlation function --  $\xi(r)$
-Codes are in xi_of_r sub-directory.
+Codes are in `xi_of_r` sub-directory.
 
 2. Measures the auto/cross-correlations between two boxes.
 Can be used to make projected correlation function -- $\xi(r_p,\pi)$
-Codes are in the xi_rp_pi sub-directory.
+Codes are in the `xi_rp_pi` sub-directory.
 
 3. Measures the full 2-point projected auto-correlation function
-in a periodic cosmological box. Codes are in the wp sub-directory.
+in a periodic cosmological box. Codes are in the `wp` sub-directory.
 
+4. Measures the full 2-point 3D auto-correlation function
+in a periodic cosmological box. Codes are in the `xi` sub-directory.
 
-*Science options*
-
-1. PERIODIC (ignored in case of wp) -- switches PERIODIC boundary
-conditions on/off. Enabled by default. 
-
-2. OUTPUT_RPAVG -- switches on output of <rp> in each bin. Can be
-a massive performance hit (~ 2.2x in case of wp). Disabled by default.
-Needs code option DOUBLE_PREC to be enabled as well. 
-
-*Code options*
-
-1. DOUBLE_PREC -- does the calculations in double precision. Disabled
-by default. 
-
-2. USE_AVX -- uses the AVX instruction set found in Intel/AMD CPUs >= 2011
-(Intel: Sandy Bridge or later; AMD: Bulldozer or later). Enabled by
-default - code will not compile if the CPU does not support AVX instructions.
-On Linux, check for "avx" in /proc/cpuinfo under flags. If you do not have
-AVX, but have a SSE4 system instead, email me - I will send you a copy of
-the code with SSE4 intrinsics. 
-
-3. USE_OMP -- uses OpenMP parallelization. Scaling is great for DD (perfect scaling
-up to 12 threads in my tests) and okay (runtime becomes constant ~6-8 threads in
-my tests) for DDrppi and wp. 
-
-*Optimization for your architecture*
-
-1. The values of bin_refine_factor and/or zbin_refine_factor in the countpairs_*.c
-files control the cache-misses, and consequently, the runtime. In my trial-and-error
-methods, I have seen any values larger than 3 are always slower. But some different
-combination of 1/2 for (z)bin_refine_factor might be faster on your platform. 
-
-2. If you have AVX2/AVX-512/KNC, you will need to rewrite the entire AVX section.
-And it is a lot of effort. Don't do it.
-
+5. Measures the counts-in-cell stats. Codes are in the `vpf` sub-directory