Merge pull request #45 from anton-matosov/conda_build

Conda build for all platforms

.travis.yml

@@ -0,0 +1,23 @@
+os:
+  - osx
+  - linux
+
+osx_image: xcode9.2
+sudo: required
+services:
+  - docker
+
+language: cpp
+
+branches:
+  only:
+    - master
+    - stable
+
+env: # Don't forget to pass env to docker in run_in_docker.sh
+  - CONDA_PY=27
+  - CONDA_PY=35
+  - CONDA_PY=36
+
+script:
+  ./travis/main.sh

CMakeLists.txt

@@ -41,15 +41,17 @@ set(SOURCE_FILES
     src/Population.cpp
     src/Population.h
     src/PythonBindings.cpp
-    src/PythonBindings.h
     src/Random.cpp
     src/Random.h
     src/Species.cpp
     src/Species.h
     src/Substrate.cpp
     src/Substrate.h
     src/Utils.cpp
-    src/Utils.h src/Traits.h src/Traits.cpp)
+    src/Utils.h
+    src/Traits.h
+    src/Traits.cpp)
+
 
 add_executable(MultiNEAT ${SOURCE_FILES})
 target_link_libraries(MultiNEAT ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} pthread)

MultiNEAT.cbp

@@ -64,7 +64,6 @@
 		<Unit filename="src/Population.cpp" />
 		<Unit filename="src/Population.h" />
 		<Unit filename="src/PythonBindings.cpp" />
-		<Unit filename="src/PythonBindings.h" />
 		<Unit filename="src/Random.cpp" />
 		<Unit filename="src/Random.h" />
 		<Unit filename="src/Species.cpp" />

MultiNEAT.project

@@ -45,7 +45,6 @@
     <File Name="src/Population.cpp"/>
     <File Name="src/Population.h"/>
     <File Name="src/PythonBindings.cpp"/>
-    <File Name="src/PythonBindings.h"/>
     <File Name="src/Random.cpp"/>
     <File Name="src/Random.h"/>
     <File Name="src/Species.cpp"/>

README.md

@@ -24,7 +24,7 @@ GNU Lesser General Public License v3.0
 * Set the required system (boost or cython) by setting an environment variable with name MN_BUILD.
 Example in Linux:
   ```bash
-  export MN_BUILD=cython
+  export MN_BUILD=boost
   ```
 
 * then, the usual:

_MultiNEAT.pyx

@@ -950,7 +950,7 @@ cdef class Species:
             del self.thisptr
 
     def __repr__(self):
-        return 'ID {} Age {}'.format(self.thisptr.ID(), self.thisptr.AgeGens())
+        return 'ID {} AgeGens {}'.format(self.thisptr.ID(), self.thisptr.AgeGens())
 
     def GetBestFitness(self):
         return self.thisptr.GetBestFitness()

appveyor.yml

@@ -0,0 +1,52 @@
+image: Visual Studio 2017
+
+# We always use a 64-bit machine, but can build x86 distributions
+# with the TARGET_ARCH variable.
+platform:
+    - x64
+
+environment:
+  fast_finish: true
+
+  matrix:
+    # Unfortunately, compiler/SDK configuration for 64 bit builds depends on
+    # python version. Right now conda build does not configure the SDK, and
+    # the appveyor setup only sets up the SDK once, so separate by python
+    # versions.
+    - TARGET_ARCH: "x64"
+      CONDA_PY: "35"
+      MINICONDA: "C:\\Miniconda%CONDA_PY%-%TARGET_ARCH%"
+    - TARGET_ARCH: "x86"
+      CONDA_PY: "35"
+      MINICONDA: "C:\\Miniconda%CONDA_PY%"
+
+    - TARGET_ARCH: "x64"
+      CONDA_PY: "36"
+      MINICONDA: "C:\\Miniconda%CONDA_PY%-%TARGET_ARCH%"
+    - TARGET_ARCH: "x86"
+      CONDA_PY: "36"
+      MINICONDA: "C:\\Miniconda%CONDA_PY%"
+
+init:
+  - "ECHO %MINICONDA%"
+
+install:
+  - "%MINICONDA%\\Scripts\\Activate.bat"
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda info -a
+  - conda install anaconda-client conda-build
+  - yes | anaconda login --username %CONDA_LOGIN_USERNAME% --password %CONDA_LOGIN_PASSWORD% --hostname MultiNEAT-APPVYR-WIN-BUILD-%APPVEYOR_BUILD_NUMBER%-%CONDA_PY%-%TARGET_ARCH%
+  - conda config --set anaconda_upload %CONDA_UPLOAD%
+
+build: false
+
+test_script:
+  - "%MINICONDA%\\Scripts\\Activate.bat"
+  - conda build conda/ -c defaults -c conda-forge
+
+skip_commits:
+  files:
+    - docs/*
+    - travis/*
+    - .travis.yml

conda/bld.bat

@@ -0,0 +1,9 @@
+@ECHO ON
+
+REM Make sure to use proper python from conda
+REM set PATH=C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\
+
+set MN_BUILD=boost
+
+python %SRC_DIR%/setup.py build_ext
+python %SRC_DIR%/setup.py install

conda/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# set -x
+set -e
+
+export MN_BUILD=boost
+
+SOURCE_CODE_DIR=${SRC_DIR:-$(dirname $0)/..}
+
+python ${SOURCE_CODE_DIR}/setup.py build_ext
+python ${SOURCE_CODE_DIR}/setup.py install

conda/conda_build_config.yaml

@@ -0,0 +1,4 @@
+python:
+    - 2.7 # [not win]
+    - 3.5
+    - 3.6

conda/meta.yaml

@@ -0,0 +1,44 @@
+package:
+  name: multineat
+  version: 0.5  # Update version in setup.py as well
+
+build:
+  number: 
+    {{ environ.get('TRAVIS_BUILD_NUMBER', environ.get('APPVEYOR_BUILD_NUMBER', 0)) }}
+
+source:
+  path: ..
+
+test:
+  source_files:
+    - examples/TestTraits.py
+    - examples/NoveltySearch.py
+    - examples/TestNEAT_xor.py
+    - examples/TestHyperNEAT_xor.py
+
+requirements:
+  build:
+    - python
+    - psutil
+    - boost=1.66*
+    - boost-cpp=1.66*
+    # - {{ compiler('cxx') }} # This results in issues with headers discovery of boost and python. On all platforms
+    # - jinja2
+  run:
+    - python
+    - psutil
+    - boost=1.66*
+    - boost-cpp=1.66*
+    - numpy>=1.11.3
+    - matplotlib=2.2*
+    - opencv=3.3*
+    # - progressbar # Python 2.7 only, needs to be replaced in main library code
+
+about:
+  home: http://MultiNEAT.com
+  license: LGPL (>= 3)
+  license_family: LGPL
+  summary: 
+    Portable NeuroEvolution Library. Implements NEAT, rtNETA, HyperNEAT, Novelty Search and others.
+  description: |
+    MultiNEAT is a portable software library for performing neuroevolution, a form of machine learning that trains neural networks with a genetic algorithm. It is based on NEAT, an advanced method for evolving neural networks through complexification. The neural networks in NEAT begin evolution with very simple genomes which grow over successive generations. The individuals in the evolving population are grouped by similarity into species, and each of them can compete only with the individuals in the same species.

conda/run_test.bat

@@ -0,0 +1,8 @@
+
+REM show test env before latering it
+set
+
+%PYTHON% "%SRC_DIR%/examples/TestTraits.py"
+REM %PYTHON% "%SRC_DIR%/examples/NoveltySearch.py"
+%PYTHON% "%SRC_DIR%/examples/TestNEAT_xor.py"
+%PYTHON% "%SRC_DIR%/examples/TestHyperNEAT_xor.py"

conda/run_test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+
+SOURCE_CODE_DIR=${SRC_DIR:-$(dirname $0)/..}
+
+# ${SOURCE_CODE_DIR}/examples/NoveltySearch.py - requires pygame and runs forever
+TESTS="${SOURCE_CODE_DIR}/examples/TestTraits.py ${SOURCE_CODE_DIR}/examples/TestNEAT_xor.py ${SOURCE_CODE_DIR}/examples/TestHyperNEAT_xor.py"
+
+echo $TESTS | xargs -n 1 -P 4 python
+

examples/TestHyperNEAT_xor.py

@@ -1,11 +1,12 @@
 #!/usr/bin/python3
+from __future__ import print_function
+
 import os
 import sys
 sys.path.insert(0, '/home/peter/code/projects/MultiNEAT') # duh
 import time
 import random as rnd
 import subprocess as comm
-import cv2
 import numpy as np
 import pickle as pickle
 import MultiNEAT as NEAT
@@ -172,7 +173,7 @@ def getbest(i):
 
 
 gens = []
-for run in range(100):
+for run in range(20):
     gen = getbest(run)
     gens += [gen]
     print('Run:', run, 'Generations to solve XOR:', gen)

examples/TestNEAT_xor.py

@@ -1,12 +1,11 @@
 #!/usr/bin/python3
-
+from __future__ import print_function
 
 import os
 import sys
 #sys.path.insert(0, '/home/peter/code/projects/MultiNEAT') # duh
 import time
 import random as rnd
-import cv2
 import numpy as np
 import pickle as pickle
 import MultiNEAT as NEAT

examples/TestTraits.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+from __future__ import print_function
 
 import time
 import random as rnd
@@ -140,7 +141,7 @@ def PrintGenomeTraits(g):
         print()
     print()
 
-for generation in range(1000):
+for generation in range(100):
 
     genome_list = NEAT.GetGenomeList(pop)
     fitness_list = NEAT.EvaluateGenomeList_Serial(genome_list, evaluate, display=False)

examples/ball_keeper.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 import sys
-sys.path.insert(0, '/home/peter/code/projects/MultiNEAT') # duh
+# sys.path.insert(0, '/home/peter/code/projects/MultiNEAT') # duh
 
 import time
 import random as rnd

setup.py

@@ -1,8 +1,31 @@
 #!/usr/bin/python
+
 from __future__ import print_function
 from setuptools import setup, Extension
 import sys
 import os
+import psutil
+
+# monkey-patch for parallel compilation
+def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None):
+    # those lines are copied from distutils.ccompiler.CCompiler directly
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+    # parallel code
+
+    N = psutil.cpu_count(logical=False) # number of parallel compilations
+    import multiprocessing.pool
+    def _single_compile(obj):
+        try: src, ext = build[obj]
+        except KeyError: return
+        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+    # convert to list, imap is evaluated on-demand
+    list(multiprocessing.pool.ThreadPool(N).imap(_single_compile,objects))
+    return objects
+
+import distutils.ccompiler
+distutils.ccompiler.CCompiler.compile=parallelCCompile
+
 
 ''' Note:
 
@@ -20,10 +43,7 @@
 
 def getExtensions():
     platform = sys.platform
-    if sys.version_info[0] < 3:
-        lb = 'boost_python'
-    else:
-        lb = 'boost_python3'  # in Ubuntu 14 there is only 'boost_python-py34'
+
     extensionsList = []
     sources = ['src/Genome.cpp',
                'src/Innovation.cpp',
@@ -37,16 +57,25 @@ def getExtensions():
                'src/Utils.cpp']
 
     extra = ['-march=native',
-             '-std=gnu++11',
-             '-g',
-             '-Wall'
+             '-g'
              ]
 
-    if 'win' in platform and platform != 'darwin':
+    if platform == 'darwin':
+        extra += ['-stdlib=libc++',
+             '-std=c++11',]
+    else:
+        extra += ['-std=gnu++11']
+
+    is_windows = 'win' in platform and platform != 'darwin'
+    if is_windows:
         extra.append('/EHsc')
     else:
         extra.append('-w')
 
+    prefix = os.getenv('PREFIX')
+    if prefix and len(prefix) > 0:
+        extra += ["-I{}/include".format(prefix)]
+
     build_sys = os.getenv('MN_BUILD')
 
     if build_sys is None:
@@ -71,9 +100,21 @@ def getExtensions():
                                                    extra_compile_args=extra)],
                                         ))
     elif build_sys == 'boost':
+        is_python_2 = sys.version_info[0] < 3
+
         sources.insert(0, 'src/PythonBindings.cpp')
-        libs = [lb, 'boost_system', 'boost_serialization']
-        # for Windows
+
+        if is_windows:
+            if is_python_2:
+                raise RuntimeError("Python prior to version 3 is not supported on Windows due to limits of VC++ compiler version")
+
+        libs = ['boost_system', 'boost_serialization']
+        if is_python_2:
+            libs += ['boost_python', "boost_numpy"]
+        else:
+            libs += ['boost_python3', "boost_numpy3"]  # in Ubuntu 14 there is only 'boost_python-py34'
+
+        # for Windows with mingw
         # libraries= ['libboost_python-mgw48-mt-1_58',
         #            'libboost_serialization-mgw48-mt-1_58'],
         # include_dirs = ['C:/MinGW/include', 'C:/Users/Peter/Desktop/boost_1_58_0'],
@@ -90,7 +131,7 @@ def getExtensions():
     return extensionsList
 
 
-setup(name='MultiNEAT',
-      version='0.5',
+setup(name='multineat',
+      version='0.5', # Update version in conda/meta.yaml as well
       packages=['MultiNEAT'],
       ext_modules=getExtensions())